diff --git a/app.rb b/app.rb index 5bcf475..12f840b 100644 --- a/app.rb +++ b/app.rb @@ -277,6 +277,33 @@ module Maptool json PmtilesBuilder.offline_status end + # Deletes offline.pmtiles plus its whole per-country download cache — + # a manual "reclaim the space"/"start over" action, distinct from + # unticking countries in the region picker (which only changes what a + # future build covers, doesn't free anything already on disk). + delete "/api/offline-tiles" do + require_login! + begin + PmtilesBuilder.delete_offline_data(DATA_DIR) + rescue PmtilesBuilder::BuildError => e + halt 409, json(error: e.message) + end + json ok: true + end + + # Deletes one country's cached Geofabrik extract only — e.g. to force a + # clean re-download of just that one, or reclaim space for a country + # that's no longer selected (the cache otherwise keeps it indefinitely). + delete "/api/offline-tiles/cache/:code" do + require_login! + begin + PmtilesBuilder.delete_cached_extract(DATA_DIR, params[:code]) + rescue PmtilesBuilder::BuildError => e + halt 409, json(error: e.message) + end + json ok: true + end + # Same optional-local-file story as offline.pmtiles above — see # docs/development.adoc for how to generate one, until then hillshade # just stays unavailable while the offline basemap is active. @@ -309,6 +336,17 @@ module Maptool json PmtilesBuilder.terrain_status end + # Same reasoning as DELETE /api/offline-tiles above. + delete "/api/terrain-tiles" do + require_login! + begin + PmtilesBuilder.delete_terrain_data(DATA_DIR) + rescue PmtilesBuilder::BuildError => e + halt 409, json(error: e.message) + end + json ok: true + end + # Pinged once per page load (public/js/map.js) — url is a same-origin # path (not kiwix-serve's own address, which the browser can't reach # directly behind the reverse proxy in front of this app), proxied by diff --git a/docs/development.adoc b/docs/development.adoc index 4641f28..7538f4b 100644 --- a/docs/development.adoc +++ b/docs/development.adoc @@ -94,12 +94,11 @@ Ticking countries and clicking "Apply changes" saves the selection (`data/offline_regions.json`, an array of ISO A2 codes) and rebuilds both files to cover it. -There's no standalone per-country file to individually delete — -`offline.pmtiles`/`terrain.pmtiles` are always one combined extract for -the *whole current selection*. "Remove a country" and "refresh a -country" are both just "change the selection, then Apply" (unticking a -country and applying rebuilds without it; nothing to separately -delete). terrain.pmtiles re-extracts its selection from scratch on +There's no standalone per-country *built* file — `offline.pmtiles`/ +`terrain.pmtiles` are always one combined extract for the *whole +current selection*. "Remove a country" is just "change the selection, +then Apply" (unticking a country and applying rebuilds without it). +terrain.pmtiles re-extracts its selection from scratch on every Apply — fine given that extraction is byte-range requests against a remote archive, not full downloads. offline.pmtiles, by contrast, downloads a whole country's `.osm.pbf` from Geofabrik (can be @@ -134,7 +133,21 @@ the current selection, `POST` to change it, then `POST /api/offline-tiles/build` / `/api/terrain-tiles/build`, polled via the matching `*/build-status` route) — only one of each kind at a time; starting a second while one's running gets a 409, as does starting -either with nothing selected yet. +either with nothing selected yet. While running, `build-status` also +includes a `step` string (e.g. `"fetching PL (3/4) — done: SK, CZ — +pending: HU"`, or `"tiling with planetiler"`) — see +`PmtilesBuilder.run_job`'s `report_step`. + +`DELETE /api/offline-tiles` removes offline.pmtiles, its sidecar meta, +and the whole per-country download cache (`data/geofabrik_cache/`) — +`DELETE /api/terrain-tiles` does the same for terrain.pmtiles. Both are +a manual "reclaim the space" / "start over" action (409 while that +kind's build is running), separate from the region picker's Apply, +which only changes what a *future* build covers and never frees +anything already on disk. `DELETE +/api/offline-tiles/cache/:code` removes just one country's cached +extract, e.g. to force a clean re-download of it without touching the +rest of the cache. Each successful build also writes a small sidecar (`offline.pmtiles. meta.json` — each source country's code/Geofabrik slug/Last-Modified; diff --git a/lib/maptool/pmtiles_builder.rb b/lib/maptool/pmtiles_builder.rb index 52720b9..a403683 100644 --- a/lib/maptool/pmtiles_builder.rb +++ b/lib/maptool/pmtiles_builder.rb @@ -189,7 +189,11 @@ module Maptool raise BuildError, "select at least one region first" if codes.empty? begin_run!(OFFLINE_MUTEX, OFFLINE_STATUS) - Thread.new { run_job(OFFLINE_MUTEX, OFFLINE_STATUS) { extract_offline(codes, data_dir, download_extract, fetch_last_modified) } } + Thread.new do + run_job(OFFLINE_MUTEX, OFFLINE_STATUS) do |report_step| + extract_offline(codes, data_dir, download_extract, fetch_last_modified, report_step) + end + end end # fetch_manifest: same injectable-seam reasoning as download_extract @@ -199,7 +203,11 @@ module Maptool raise BuildError, "select at least one region first" if codes.empty? begin_run!(TERRAIN_MUTEX, TERRAIN_STATUS) - Thread.new { run_job(TERRAIN_MUTEX, TERRAIN_STATUS) { build_terrain_pmtiles(codes, data_dir, fetch_manifest) } } + Thread.new do + run_job(TERRAIN_MUTEX, TERRAIN_STATUS) do |report_step| + build_terrain_pmtiles(codes, data_dir, fetch_manifest, report_step) + end + end end def self.begin_run!(mutex, status) @@ -210,11 +218,19 @@ module Maptool end private_class_method :begin_run! + # Yields a report_step proc the build can call with a short + # human-readable description of what it's doing right now ("downloading + # SK (1/2)", "tiling with planetiler", ...) — surfaced as status[:step] + # so GET .../build-status can show progress beyond just "running since + # X" (see docs/development.adoc). def self.run_job(mutex, status) - yield + report_step = ->(text) { mutex.synchronize { status[:step] = text } } + yield report_step mutex.synchronize { status.merge!(state: "done", finished_at: Time.now.utc.iso8601).delete(:message) } rescue => e mutex.synchronize { status.merge!(state: "error", finished_at: Time.now.utc.iso8601, message: e.message) } + ensure + mutex.synchronize { status.delete(:step) } end private_class_method :run_job @@ -228,10 +244,11 @@ module Maptool # tiles.openfreemap.org's Liberty style already expects, so # public/js/map.js's offlineStyle() can reuse ONLINE_STYLE_PATCH # unchanged instead of a separate protomaps-themes-base palette. - def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified) + def self.extract_offline(codes, data_dir, download_extract, fetch_last_modified, report_step) scratch_dir = Dir.mktmpdir("maptool-offline") - sources = codes.map do |code| + sources = codes.each_with_index.map do |code, i| slug = GEOFABRIK_SLUGS.fetch(code) { raise BuildError, "no Geofabrik source known for region #{code}" } + report_step.call(progress_step("fetching #{code}", codes, i)) cached_extract(code, slug, data_dir, download_extract, fetch_last_modified) end @@ -239,12 +256,18 @@ module Maptool if sources.size == 1 sources.first["path"] else + report_step.call("merging with osmium") merged_path = File.join(scratch_dir, "merged.osm.pbf") run_osmium!("merge", *sources.map { |s| s["path"] }, "-o", merged_path) merged_path end - tmp_path = File.join(data_dir, "offline.pmtiles.tmp") + report_step.call("tiling with planetiler") + # .tmp.pmtiles, not .pmtiles.tmp — Planetiler picks its output archive + # format (pmtiles vs mbtiles) from the file extension, so a ".tmp" + # suffix made it fail with "Unsupported format tmp" instead of + # actually tiling anything. + tmp_path = File.join(data_dir, "offline.tmp.pmtiles") run_planetiler!("--osm-path=#{osm_path}", "--output=#{tmp_path}") File.rename(tmp_path, File.join(data_dir, "offline.pmtiles")) File.write(offline_meta_path(data_dir), JSON.generate(sources: sources.map { |s| s.slice("code", "slug", "last_modified") })) @@ -254,6 +277,20 @@ module Maptool end private_class_method :extract_offline + # Renders a report_step string like "fetching PL (3/4) — done: SK, CZ — + # pending: HU" — so GET .../build-status shows not just "it's running" + # but which of the current build's sources is in flight and which are + # already behind/ahead of it. + def self.progress_step(action, labels, index) + done = labels[0...index] + pending = labels[(index + 1)..] || [] + parts = ["#{action} (#{index + 1}/#{labels.size})"] + parts << "done: #{done.join(", ")}" unless done.empty? + parts << "pending: #{pending.join(", ")}" unless pending.empty? + parts.join(" — ") + end + private_class_method :progress_step + def self.geofabrik_url(slug) "https://download.geofabrik.de/#{slug}-latest.osm.pbf" end @@ -378,6 +415,31 @@ module Maptool end private_class_method :offline_meta_path + # Deletes offline.pmtiles, its sidecar meta, and the whole per-country + # download cache (data/geofabrik_cache/) — "start over" for the offline + # basemap. The current selection (data/offline_regions.json) is left + # alone, so a following build just re-downloads it fresh. + def self.delete_offline_data(data_dir) + OFFLINE_MUTEX.synchronize { raise BuildError, "a build is already running" if OFFLINE_STATUS[:state] == "running" } + + File.delete(File.join(data_dir, "offline.pmtiles")) if File.exist?(File.join(data_dir, "offline.pmtiles")) + File.delete(offline_meta_path(data_dir)) if File.exist?(offline_meta_path(data_dir)) + cache_dir = geofabrik_cache_dir(data_dir) + FileUtils.remove_entry(cache_dir) if Dir.exist?(cache_dir) + end + + # Deletes just one country's cached .osm.pbf (and its Last-Modified + # stamp) — e.g. to force a fresh download without touching the others, + # or to reclaim space for a country no longer selected. + def self.delete_cached_extract(data_dir, code) + OFFLINE_MUTEX.synchronize { raise BuildError, "a build is already running" if OFFLINE_STATUS[:state] == "running" } + + cache_dir = geofabrik_cache_dir(data_dir) + path = File.join(cache_dir, "#{code}.osm.pbf") + File.delete(path) if File.exist?(path) + File.delete("#{path}.last_modified") if File.exist?("#{path}.last_modified") + end + # --- terrain.pmtiles --- # planet.pmtiles (z0-12, always) plus whichever regional chunks @@ -394,7 +456,8 @@ module Maptool # border tiles (extraction copies whole tiles, it doesn't clip vector # content to the polygon) — so this always does one extract per # *source archive*, never one per country. - def self.build_terrain_pmtiles(codes, data_dir, fetch_manifest) + def self.build_terrain_pmtiles(codes, data_dir, fetch_manifest, report_step) + report_step.call("fetching mapterhorn manifest") items = fetch_manifest.call planet = items.find { |item| item["name"] == MAPTERHORN_PLANET_NAME } raise BuildError, "mapterhorn manifest is missing #{MAPTERHORN_PLANET_NAME}" unless planet @@ -407,8 +470,14 @@ module Maptool chunks = items.select { |item| item["name"] != MAPTERHORN_PLANET_NAME && bbox_intersects?(item, south, west, north, east) } - tmp_path = File.join(data_dir, "terrain.pmtiles.tmp") - extracts = [planet, *chunks].each_with_index.map do |item, i| + # .tmp.pmtiles, not .pmtiles.tmp — same reasoning as offline.pmtiles' + # own tmp_path above, in case go-pmtiles also derives format from + # the extension for its "merge" output. + tmp_path = File.join(data_dir, "terrain.tmp.pmtiles") + sources = [planet, *chunks] + labels = sources.map { |item| item["name"] } + extracts = sources.each_with_index.map do |item, i| + report_step.call(progress_step("extracting #{item["name"]}", labels, i)) extract_path = File.join(scratch_dir, "extract-#{i}.pmtiles") run_pmtiles!("extract", item["url"], extract_path, "--region=#{region_path}", "--maxzoom=#{TERRAIN_MAX_ZOOM}") extract_path @@ -417,6 +486,7 @@ module Maptool if extracts.size == 1 File.rename(extracts.first, tmp_path) else + report_step.call("merging with pmtiles") run_pmtiles!("merge", *extracts, tmp_path) end File.rename(tmp_path, File.join(data_dir, "terrain.pmtiles")) @@ -479,6 +549,17 @@ module Maptool end private_class_method :terrain_meta_path + # Deletes terrain.pmtiles and its sidecar meta — same "start over" + # reasoning as delete_offline_data above. No download cache to clean up + # here: terrain.pmtiles' pmtiles-extract step reads its sources over + # HTTP range requests rather than downloading a whole archive first. + def self.delete_terrain_data(data_dir) + TERRAIN_MUTEX.synchronize { raise BuildError, "a build is already running" if TERRAIN_STATUS[:state] == "running" } + + File.delete(File.join(data_dir, "terrain.pmtiles")) if File.exist?(File.join(data_dir, "terrain.pmtiles")) + File.delete(terrain_meta_path(data_dir)) if File.exist?(terrain_meta_path(data_dir)) + end + # --- shared --- def self.read_json(path) diff --git a/public/css/style.css b/public/css/style.css index 4976cc1..4679802 100644 --- a/public/css/style.css +++ b/public/css/style.css @@ -472,6 +472,33 @@ html, body { border-top: 1px solid #eee; } +.offline-data-panel__row-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; +} + +.offline-data-panel__delete { + border: 1px solid #ccc; + border-radius: 4px; + background: #fff; + padding: 2px 8px; + font: inherit; + font-size: 11px; + color: #a33; + cursor: pointer; +} + +.offline-data-panel__delete:hover { + background: #fdecec; +} + +.offline-data-panel__delete:disabled { + opacity: 0.5; + cursor: default; +} + .offline-data-panel__status { margin: 6px 0 0; color: #666; diff --git a/public/js/offline_data.js b/public/js/offline_data.js index 5cee5da..b4e2852 100644 --- a/public/js/offline_data.js +++ b/public/js/offline_data.js @@ -17,12 +17,14 @@ function formatElapsed(ms) { return minutes > 0 ? `${minutes}m ${seconds}s` : `${seconds}s`; } -// "Building…" alone gave no way to tell a slow build from a stuck one — -// elapsed time answers "is it alive" regardless of how long a given -// selection takes. +// "Building…" alone gave no way to tell a slow build from a stuck one, or +// which of the current selection's sources it's actually working on right +// now — elapsed time answers "is it alive", status.step (see +// PmtilesBuilder.run_job) answers "doing what, and what's left". function formatRunning(status) { const elapsed = status.started_at ? ` (${formatElapsed(Date.now() - new Date(status.started_at).getTime())})` : ""; - return `Building…${elapsed}`; + const step = status.step ? `: ${status.step}` : ""; + return `Building…${elapsed}${step}`; } // Vendored square-cropped SVGs (public/vendor/flag-icons/, MIT, see @@ -131,9 +133,10 @@ class RegionPicker { // one class covers both rather than writing the same fetch/poll logic // twice. class DataFileRow { - constructor({ label, statusUrl, buildUrl, buildStatusUrl, checkUpdateUrl, formatMeta }) { + constructor({ label, statusUrl, deleteUrl, buildUrl, buildStatusUrl, checkUpdateUrl, formatMeta }) { this._label = label; this._statusUrl = statusUrl; + this._deleteUrl = deleteUrl; this._buildUrl = buildUrl; this._buildStatusUrl = buildStatusUrl; this._checkUpdateUrl = checkUpdateUrl; @@ -144,10 +147,15 @@ class DataFileRow { const row = document.createElement("div"); row.className = "offline-data-panel__row"; row.innerHTML = ` - ${this._label} +
+ ${this._label} + +

Checking…

`; this._statusEl = row.querySelector(".offline-data-panel__status"); + this._deleteButton = row.querySelector(".offline-data-panel__delete"); + this._deleteButton.addEventListener("click", () => this._delete()); container.appendChild(row); this._refreshStatus(); } @@ -163,14 +171,39 @@ class DataFileRow { if (!data.available) { this._statusEl.textContent = "Not generated yet."; + this._deleteButton.hidden = true; return; } const updated = new Date(data.mtime).toLocaleString([], { dateStyle: "short", timeStyle: "short" }); this._statusEl.textContent = `${formatBytes(data.size)} · updated ${updated}${this._formatMeta(data.meta)}`; + this._deleteButton.hidden = !isLoggedIn(); this._checkUpdate(); } + // Deletes the built file (and, for offline.pmtiles, its whole per-country + // download cache — see PmtilesBuilder.delete_offline_data) so the row + // goes back to "Not generated yet." — a manual way to reclaim space or + // force a completely fresh rebuild, since Apply alone never removes + // anything already on disk. + async _delete() { + if (!window.confirm(`Delete ${this._label.toLowerCase()}? This can't be undone.`)) return; + + this._deleteButton.disabled = true; + try { + const response = await fetch(this._deleteUrl, { method: "DELETE" }); + const data = await response.json().catch(() => ({})); + if (!response.ok) throw new Error(data.error || `could not delete: ${response.status}`); + } catch (err) { + window.alert(err.message); + return; + } finally { + this._deleteButton.disabled = false; + } + + this._refreshStatus(); + } + // Separate from _refreshStatus's own fetch (and only called once, right // after — not polled): this one reaches an external service (Geofabrik // or Mapterhorn's manifest, see lib/maptool/pmtiles_builder.rb), so it's @@ -249,6 +282,7 @@ export class OfflineDataControl { new DataFileRow({ label: "Offline map data", statusUrl: "/api/offline-tiles", + deleteUrl: "/api/offline-tiles", buildUrl: "/api/offline-tiles/build", buildStatusUrl: "/api/offline-tiles/build-status", checkUpdateUrl: "/api/offline-tiles/check-update", @@ -257,6 +291,7 @@ export class OfflineDataControl { new DataFileRow({ label: "Offline terrain data", statusUrl: "/api/terrain-tiles", + deleteUrl: "/api/terrain-tiles", buildUrl: "/api/terrain-tiles/build", buildStatusUrl: "/api/terrain-tiles/build-status", checkUpdateUrl: "/api/terrain-tiles/check-update", diff --git a/test/test_app.rb b/test/test_app.rb index 972e242..53795aa 100644 --- a/test/test_app.rb +++ b/test/test_app.rb @@ -72,6 +72,9 @@ class AppTest < Minitest::Test assert_equal 401, request.post("/api/plugins/test-plugin/run").status assert_equal 401, request.post("/api/offline-tiles/build").status assert_equal 401, request.post("/api/terrain-tiles/build").status + assert_equal 401, request.delete("/api/offline-tiles").status + assert_equal 401, request.delete("/api/offline-tiles/cache/SK").status + assert_equal 401, request.delete("/api/terrain-tiles").status end # --- GPX/FIT upload and the tree --- @@ -417,6 +420,35 @@ class AppTest < Minitest::Test thread&.join end + def test_offline_tiles_delete_is_ok_when_nothing_was_built + assert_equal 200, request.delete("/api/offline-tiles", **logged_in_env).status + end + + def test_offline_tiles_delete_removes_the_file_and_cache + path = File.join(Maptool::App::DATA_DIR, "offline.pmtiles") + cache_dir = File.join(Maptool::App::DATA_DIR, "geofabrik_cache") + File.write(path, "fake") + FileUtils.mkdir_p(cache_dir) + File.write(File.join(cache_dir, "SK.osm.pbf"), "fake") + + response = request.delete("/api/offline-tiles", **logged_in_env) + + assert_equal 200, response.status + refute_path_exists path + refute_path_exists cache_dir + end + + def test_offline_tiles_delete_is_409_when_a_build_is_running + Maptool::PmtilesBuilder.set_selected_regions(Maptool::App::DATA_DIR, ["SK"]) + gate = Queue.new + thread = Maptool::PmtilesBuilder.build_offline(data_dir: Maptool::App::DATA_DIR, download_extract: ->(_url, _dest) { gate.pop }) + + assert_equal 409, request.delete("/api/offline-tiles", **logged_in_env).status + ensure + gate.push("unused") + thread&.join + end + # --- LiveTrack webhook --- def test_livetrack_webhook_requires_the_configured_secret diff --git a/test/test_pmtiles_builder.rb b/test/test_pmtiles_builder.rb index d91bcb7..0a6328b 100644 --- a/test/test_pmtiles_builder.rb +++ b/test/test_pmtiles_builder.rb @@ -171,7 +171,7 @@ class PmtilesBuilderTest < Minitest::Test thread.join assert_path_exists File.join(@data_dir, "offline.pmtiles") - refute_path_exists File.join(@data_dir, "offline.pmtiles.tmp") + refute_path_exists File.join(@data_dir, "offline.tmp.pmtiles") assert_equal "done", Maptool::PmtilesBuilder.offline_status[:state] assert_equal ["planetiler --osm-path"], log_lines assert_equal( @@ -260,6 +260,84 @@ class PmtilesBuilderTest < Minitest::Test assert_equal 1, calls.count { |url| url.include?("austria") } end + # offline_status[:step] should say which country is in flight right now + # and which of the current build's selection are already done/still + # pending — not just "running since X". + def test_build_offline_reports_progress_step_while_running + select_regions("SK", "AT") + started = Queue.new + gate = Queue.new + download_extract = ->(url, dest) { + started.push(url) + gate.pop + File.write(dest, "fake") + "Wed, 10 Sep 2026 08:00:01 GMT" + } + thread = Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: download_extract) + + started.pop + assert_equal "fetching SK (1/2) — pending: AT", Maptool::PmtilesBuilder.offline_status[:step] + gate.push(:go) + + started.pop + assert_equal "fetching AT (2/2) — done: SK", Maptool::PmtilesBuilder.offline_status[:step] + gate.push(:go) + + thread.join + assert_nil Maptool::PmtilesBuilder.offline_status[:step] + end + + def test_delete_offline_data_removes_the_file_meta_and_cache + select_regions("SK") + Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download).join + assert_path_exists File.join(@data_dir, "offline.pmtiles") + assert_path_exists File.join(@data_dir, "geofabrik_cache", "SK.osm.pbf") + + Maptool::PmtilesBuilder.delete_offline_data(@data_dir) + + refute_path_exists File.join(@data_dir, "offline.pmtiles") + refute_path_exists File.join(@data_dir, "geofabrik_cache") + assert_nil Maptool::PmtilesBuilder.offline_meta(@data_dir) + end + + def test_delete_offline_data_is_a_noop_when_nothing_was_built + Maptool::PmtilesBuilder.delete_offline_data(@data_dir) # should not raise + end + + def test_delete_offline_data_raises_while_a_build_is_running + select_regions("SK") + gate = Queue.new + thread = Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: ->(_url, _dest) { gate.pop }) + + assert_raises(Maptool::PmtilesBuilder::BuildError) { Maptool::PmtilesBuilder.delete_offline_data(@data_dir) } + ensure + gate.push("unused") + thread&.join + end + + def test_delete_cached_extract_removes_only_that_country + select_regions("SK", "AT") + Maptool::PmtilesBuilder.build_offline(data_dir: @data_dir, download_extract: fake_download).join + + Maptool::PmtilesBuilder.delete_cached_extract(@data_dir, "SK") + + refute_path_exists File.join(@data_dir, "geofabrik_cache", "SK.osm.pbf") + assert_path_exists File.join(@data_dir, "geofabrik_cache", "AT.osm.pbf") + end + + def test_delete_terrain_data_removes_the_file_and_meta + select_regions("ES") + ENV["PMTILES_BIN"] = @pmtiles_ok + thread = Maptool::PmtilesBuilder.build_terrain(data_dir: @data_dir, fetch_manifest: -> { fake_manifest }) + thread.join + assert_path_exists File.join(@data_dir, "terrain.pmtiles") + + Maptool::PmtilesBuilder.delete_terrain_data(@data_dir) + + refute_path_exists File.join(@data_dir, "terrain.pmtiles") + assert_nil Maptool::PmtilesBuilder.terrain_meta(@data_dir) + end + def test_offline_meta_is_nil_before_any_build assert_nil Maptool::PmtilesBuilder.offline_meta(@data_dir) end