From fc174d42673d73b89a5fcaed0bee17e464d4b250 Mon Sep 17 00:00:00 2001 From: Roman Priesol Date: Mon, 7 Sep 2026 12:23:37 +0200 Subject: [PATCH] Support uploading FIT files (and zipped exports) alongside GPX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Maptool::Fit reads Garmin's binary FIT format directly rather than adding a gem — the one Ruby option depends on activesupport, unmaintained since 2022, for a single-purpose binary parser. Only "record" messages are read (position + altitude; everything else is skipped by byte length, never interpreted); field numbers and scale/offset are Garmin's own (fit-python-sdk's profile.py), and the framing plus all extracted values were checked byte-for-byte against 5 real device files from python-fitparse's test suite before this shipped. A .zip upload (Garmin's bulk/FIT export format) is expanded into its .gpx/.fit entries via rubyzip — that one's an actual dependency, since ZIP's own format isn't the kind of narrow, single-purpose parsing FIT above is. --- Gemfile | 1 + Gemfile.lock | 3 + app.rb | 35 +++++- docs/architecture.adoc | 66 ++++++++--- lib/maptool/fit.rb | 232 +++++++++++++++++++++++++++++++++++++++ lib/maptool/gpx_store.rb | 17 ++- public/js/gpx.js | 7 +- views/index.erb | 6 +- 8 files changed, 338 insertions(+), 29 deletions(-) create mode 100644 lib/maptool/fit.rb diff --git a/Gemfile b/Gemfile index 24a7146..1cbccf3 100644 --- a/Gemfile +++ b/Gemfile @@ -5,6 +5,7 @@ gem "sinatra-contrib", "~> 4.2" gem "puma", "~> 8.0" gem "rexml", "~> 3.4" gem "oauth2", "~> 2.0" +gem "rubyzip", "~> 2.4" group :development do gem "rerun" diff --git a/Gemfile.lock b/Gemfile.lock index 4788232..a6c671d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -66,6 +66,7 @@ GEM rerun (0.14.0) listen (~> 3.0) rexml (3.4.4) + rubyzip (2.4.1) sinatra (4.2.1) logger (>= 1.6.0) mustermann (~> 3.0) @@ -103,6 +104,7 @@ DEPENDENCIES puma (~> 8.0) rerun rexml (~> 3.4) + rubyzip (~> 2.4) sinatra (~> 4.2) sinatra-contrib (~> 4.2) @@ -143,6 +145,7 @@ CHECKSUMS rb-inotify (0.11.1) sha256=a0a700441239b0ff18eb65e3866236cd78613d6b9f78fea1f9ac47a85e47be6e rerun (0.14.0) sha256=996a5af2b2b34dcb30146c6c728addab9113e07c119003370daa9bf27feb88fd rexml (3.4.4) sha256=19e0a2c3425dfbf2d4fc1189747bdb2f849b6c5e74180401b15734bc97b5d142 + rubyzip (2.4.1) sha256=8577c88edc1fde8935eb91064c5cb1aef9ad5494b940cf19c775ee833e075615 sinatra (4.2.1) sha256=b7aeb9b11d046b552972ade834f1f9be98b185fa8444480688e3627625377080 sinatra-contrib (4.2.1) sha256=10d091c944d268aa910c618ea40a3c3ebe0533e6e32990d84af92235a3d26b4a snaky_hash (2.0.7) sha256=7d02c70012a3f932e48860cd024577908300c9aa615e0cb9b450aaa749cbcb4d diff --git a/app.rb b/app.rb index 7ac5335..e4edd83 100644 --- a/app.rb +++ b/app.rb @@ -2,7 +2,10 @@ require "sinatra/base" require "sinatra/json" require "oauth2" require "securerandom" +require "zip" +require "stringio" require_relative "lib/maptool/gpx" +require_relative "lib/maptool/fit" require_relative "lib/maptool/gpx_store" require_relative "lib/maptool/gpsd_client" require_relative "lib/maptool/livetrack_store" @@ -69,6 +72,18 @@ module Maptool def callback_url "#{request.base_url}/auth/callback" end + + def save_zip_entries(folder:, io:) + paths = [] + Zip::File.open_buffer(io) do |zip| + zip.each do |entry| + next unless entry.file? && GpxStore::TRACK_EXTENSIONS.any? { |ext| entry.name.downcase.end_with?(ext) } + + paths << GPX_STORE.save(folder: folder, filename: File.basename(entry.name), io: StringIO.new(entry.get_input_stream.read)) + end + end + paths + end end get "/login" do @@ -149,18 +164,27 @@ module Maptool json GPX_STORE.tree end + # A .zip (Garmin's bulk/FIT export format) is expanded into its + # .gpx/.fit entries instead of being stored itself — nothing downstream + # (tree listing, map rendering) knows what to do with a zip. post "/api/gpx" do require_login! file = params[:file] halt 400, json(error: "file is required") unless file && file[:tempfile] begin - path = GPX_STORE.save(folder: params[:folder].to_s, filename: file[:filename], io: file[:tempfile]) + paths = if file[:filename].to_s.downcase.end_with?(".zip") + save_zip_entries(folder: params[:folder].to_s, io: file[:tempfile]) + else + [GPX_STORE.save(folder: params[:folder].to_s, filename: file[:filename], io: file[:tempfile])] + end rescue GpxStore::InvalidPath halt 400, json(error: "invalid folder") + rescue Zip::Error + halt 400, json(error: "invalid zip file") end - json path: path, tree: GPX_STORE.tree + json paths: paths, tree: GPX_STORE.tree end get "/api/gpx/file/*" do @@ -172,9 +196,10 @@ module Maptool halt 404 unless File.file?(path) begin - json Gpx.to_geojson(path) - rescue Gpx::ParseError - halt 422, json(error: "could not parse GPX file") + geojson = path.downcase.end_with?(".fit") ? Fit.to_geojson(path) : Gpx.to_geojson(path) + json geojson + rescue Gpx::ParseError, Fit::ParseError + halt 422, json(error: "could not parse file") end end diff --git a/docs/architecture.adoc b/docs/architecture.adoc index 902c69a..40e25e2 100644 --- a/docs/architecture.adoc +++ b/docs/architecture.adoc @@ -31,10 +31,10 @@ opened in a browser. | `data/` | Local, gitignored data files, e.g. `offline.pmtiles` and uploaded - GPX files under `data/gpx/`. + GPX/FIT files under `data/gpx/`. | `lib/maptool/` -| Application code beyond routing: the GPX parser and file store. +| Application code beyond routing: the GPX/FIT parsers and file store. | `docs/` | Project documentation, in AsciiDoc. @@ -134,11 +134,12 @@ This is a live, online-only overlay — it always needs internet access and is unrelated to the offline PMTiles basemap above. Per RainViewer's terms, the API is free for personal/non-commercial use. -== GPX tracks and waypoints +== GPX/FIT tracks and waypoints -The "GPX" map control (`public/js/gpx.js`) opens a side panel to -upload GPX files by dragging them in, organize them into folders (also -by dragging), and toggle each one on the map individually. +The "Tracks" map control (`public/js/gpx.js`) opens a side panel to +upload GPX or FIT files by dragging them in, organize them into +folders (also by dragging), and toggle each one on the map +individually. Storage:: Files are stored as-is under `data/gpx/`, in a folder hierarchy the user manages directly — `Maptool::GpxStore` @@ -155,8 +156,13 @@ which the panel renders recursively (folders as `
`, files as a checkbox, both with a delete button). Drag & drop:: The dropzone at the top of the panel accepts one or more -`.gpx` files (`POST /api/gpx`, multipart) into the top level, or click -it to browse instead. An already-uploaded file (a +`.gpx`/`.fit` files, or a `.zip` (`POST /api/gpx`, multipart) into the +top level, or click it to browse instead. A `.zip` — Garmin's bulk/FIT +export format — isn't stored itself: `save_zip_entries` (`app.rb`) +expands it with `rubyzip` and saves each `.gpx`/`.fit` entry it +contains individually (flattened — any folder structure inside the +zip is discarded), skipping anything else (readme files, etc.) it +might hold. An already-uploaded file (a `.gpx-tree__file` row, `draggable`) can be dragged onto a folder to move it there (`POST /api/gpx/move`), or dropped anywhere else in the tree to move it back to the top level; an OS file can also be dropped @@ -168,16 +174,46 @@ map) gets moved, its map layers are re-added under the new path so it stays visible; if it (or its containing folder) gets deleted, its layers are removed instead. -Rendering:: Checking a file fetches `GET -/api/gpx/file/` — `Maptool::Gpx` (`lib/maptool/gpx.rb`) parses -the GPX with REXML and converts it to a GeoJSON `FeatureCollection`: -`` waypoints become `Point` features (with `name`, `desc` and -`time` carried through as properties), each `` becomes one -`MultiLineString` feature (one line per ``). MapLibre's +Rendering:: Checking a file fetches `GET /api/gpx/file/`, routed +by extension to `Maptool::Gpx` or `Maptool::Fit` +(`lib/maptool/gpx.rb`/`fit.rb`) for parsing, and both convert to the +same GeoJSON `FeatureCollection` shape so the frontend doesn't need to +know which format it got. `Gpx` parses the GPX with REXML: `` +waypoints become `Point` features (with `name`, `desc` and `time` +carried through as properties), each `` becomes one +`MultiLineString` feature (one line per ``); `` routes +are not handled. `Fit` reads Garmin's binary FIT format directly +(hand-rolled — see below) into a single `MultiLineString` `track` +feature; FIT activity files don't really have a GPX-waypoint +equivalent, so it never produces `Point` features. MapLibre's `geometry-type` expression treats `MultiLineString` the same as `LineString`, so a single GeoJSON source per file with a `line` layer and a `circle` layer (each filtered by geometry type) is enough to -render both waypoints and tracks. `` routes are not handled. +render both. + +FIT parsing:: No gem was added for this — the one Ruby option +(`fit-parser`, tjwallace/fit) hasn't been touched since 2022 and pulls +in `activesupport` for a single-purpose parser, a heavy, stale +dependency for what this app actually needs. `Maptool::Fit` instead +reads only "record" messages (global message number 20 — every other +message type, laps/sessions/events/device info/developer fields, is +skipped by byte length by summing each field's declared size, never +interpreted) for 3 fields: `position_lat`/`position_long` (field 0/1, +`sint32` semicircles, converted to degrees by `* (180 / 2**31)`) and +`altitude`/`enhanced_altitude` (field 2/78, scale 5 / offset 500 — +`raw / 5.0 - 500`, in meters). Field numbers and scale/offset are +Garmin's own (github.com/garmin/fit-python-sdk's `profile.py`), not +guessed. Both the message framing (headers, definition messages, +compressed-timestamp headers, little/big-endian) and the 3 fields' +values were checked against `github.com/dtcooper/python-fitparse`'s +own test fixtures — 5 real device files spanning 2013-2019, one with +developer fields, one using only compressed-timestamp headers — byte +after byte, all coordinates and altitudes exact, before this shipped. + +`.zip` extraction does use a gem (`rubyzip`) — parsing the ZIP central +directory format correctly (including compressed entries) isn't the +kind of narrowly-scoped, single-purpose logic FIT parsing above is, +and rubyzip is the de facto standard for this in Ruby. Like the radar overlay, GPX layers are removed by `map.setStyle()` (switching Online/Offline) and get re-added on `style.load` for diff --git a/lib/maptool/fit.rb b/lib/maptool/fit.rb new file mode 100644 index 0000000..d558b0e --- /dev/null +++ b/lib/maptool/fit.rb @@ -0,0 +1,232 @@ +module Maptool + # Converts a Garmin FIT file (binary — see https://developer.garmin.com/fit/) + # to a GeoJSON FeatureCollection, in the same shape Gpx.to_geojson produces + # for a : one Feature per file, geometry MultiLineString, "kind" => + # "track". Only positioned "record" messages are read — FIT also carries + # laps/sessions/events/device info/developer fields, none of which this + # app has anywhere to show, so they're skipped by byte-length rather than + # interpreted. + module Fit + class ParseError < StandardError; end + + FIT_EPOCH = Time.utc(1989, 12, 31).to_i + RECORD_MESSAGE_NUM = 20 + FIELD_POSITION_LAT = 0 + FIELD_POSITION_LONG = 1 + FIELD_ALTITUDE = 2 + FIELD_ENHANCED_ALTITUDE = 78 + SEMICIRCLE_TO_DEGREES = 180.0 / (2**31) + + Definition = Struct.new(:global_mesg_num, :little_endian, :fields, :total_size) + + def self.kinds(path) + to_geojson(path)["features"].empty? ? [] : ["track"] + rescue ParseError + [] + end + + def self.to_geojson(path) + points = parse(File.binread(path)) + features = points.empty? ? [] : [track_feature(points)] + { "type" => "FeatureCollection", "features" => features } + end + + def self.track_feature(points) + { + "type" => "Feature", + "geometry" => { "type" => "MultiLineString", "coordinates" => [points] }, + "properties" => { "kind" => "track", "name" => nil }, + } + end + private_class_method :track_feature + + def self.parse(data) + raise ParseError, "not a FIT file" if data.bytesize < 12 || data.byteslice(8, 4) != ".FIT" + + header_size = data.getbyte(0) + data_size = data.byteslice(4, 4).unpack1("L<") + data_end = header_size + data_size + raise ParseError, "truncated file" if data_end > data.bytesize + + offset = header_size + definitions = {} + points = [] + + while offset < data_end + header = data.getbyte(offset) + offset += 1 + + if header & 0x80 != 0 + local_type = (header >> 5) & 0x3 + definition = definitions[local_type] or raise ParseError, "data message before its definition" + offset = read_data_message(data, offset, definition, points) + elsif header & 0x40 != 0 + local_type = header & 0x0F + definition, offset = read_definition(data, offset, header & 0x20 != 0) + definitions[local_type] = definition + else + local_type = header & 0x0F + definition = definitions[local_type] or raise ParseError, "data message before its definition" + offset = read_data_message(data, offset, definition, points) + end + end + + points + rescue ArgumentError, NoMethodError => e + raise ParseError, e.message + end + private_class_method :parse + + def self.read_definition(data, offset, has_developer_fields) + architecture = data.getbyte(offset + 1) + little_endian = architecture.zero? + global_mesg_num = data.byteslice(offset + 2, 2).unpack1(little_endian ? "S<" : "S>") + num_fields = data.getbyte(offset + 4) + offset += 5 + + fields = [] + num_fields.times do + fields << [data.getbyte(offset), data.getbyte(offset + 1)] + offset += 3 + end + + if has_developer_fields + num_dev_fields = data.getbyte(offset) + offset += 1 + num_dev_fields.times do + fields << [nil, data.getbyte(offset + 1)] + offset += 3 + end + end + + total_size = fields.sum { |_field_num, size| size } + [Definition.new(global_mesg_num, little_endian, fields, total_size), offset] + end + private_class_method :read_definition + + def self.read_data_message(data, offset, definition, points) + if definition.global_mesg_num == RECORD_MESSAGE_NUM + point = extract_point(data, offset, definition) + points << point if point + end + offset + definition.total_size + end + private_class_method :read_data_message + + # Only reads the 3 fields the map needs; every other field in the + # message (heart rate, cadence, power, ...) is walked past via its + # declared size without being interpreted. + def self.extract_point(data, offset, definition) + lat = lon = altitude = nil + pos = offset + + definition.fields.each do |field_num, size| + case field_num + when FIELD_POSITION_LAT + lat = unpack_int(data, pos, size, definition.little_endian, signed: true) + when FIELD_POSITION_LONG + lon = unpack_int(data, pos, size, definition.little_endian, signed: true) + when FIELD_ALTITUDE, FIELD_ENHANCED_ALTITUDE + altitude ||= unpack_int(data, pos, size, definition.little_endian, signed: false) + end + pos += size + end + + return nil if lat.nil? || lon.nil? || invalid?(lat, 4) || invalid?(lon, 4) + + coordinate = [lon * SEMICIRCLE_TO_DEGREES, lat * SEMICIRCLE_TO_DEGREES] + coordinate << (altitude / 5.0 - 500) if altitude && !invalid?(altitude, 2) && !invalid?(altitude, 4) + coordinate + end + private_class_method :extract_point + + UNPACK_FORMATS = { + [1, true] => "c", [1, false] => "C", + [2, true] => "s", [2, false] => "S", + [4, true] => "l", [4, false] => "L", + [8, true] => "q", [8, false] => "Q", + }.freeze + + def self.unpack_int(data, offset, size, little_endian, signed:) + format = UNPACK_FORMATS[[size, signed]] + return nil unless format + + format += little_endian ? "<" : ">" if size > 1 + data.byteslice(offset, size)&.unpack1(format) + end + private_class_method :unpack_int + + # FIT's per-base-type "field not present" sentinel: all bits set for an + # unsigned/positive-only reading, the same pattern for signed. Good + # enough for the two fields this cares about (position, altitude) + # without needing every base type's exact invalid value. + def self.invalid?(value, size) + value == (1 << (size * 8 - 1)) - 1 || value == -1 || value == (1 << (size * 8)) - 1 + end + private_class_method :invalid? + end +end + +if __FILE__ == $0 + require "tmpdir" + + # A hand-built minimal FIT file: header + one definition message (local + # type 0, global mesg 20 "record", fields position_lat/position_long, + # both sint32) + two data messages + a throwaway CRC. Exercises the + # header/definition/data framing without external fixtures. + def le32(n) + [n].pack("l<") + end + + def build_fit(records) + definition = [0, 0].pack("CC") + [20].pack("S<") + [2].pack("C") + + [0, 4, 0x85].pack("C3") + [1, 4, 0x85].pack("C3") + definition_header = [0b01000000].pack("C") + + data = records.map do |lat, lon| + [0b00000000].pack("C") + le32(lat) + le32(lon) + end.join + + body = definition_header + definition + data + header = [12, 16, 0, 0].pack("C2S<") + le32(body.bytesize) + ".FIT" + header + body + "\x00\x00" + end + + Dir.mktmpdir do |dir| + path = File.join(dir, "test.fit") + File.binwrite(path, build_fit([[100_000_000, 200_000_000], [100_000_100, 200_000_100]])) + + geojson = Maptool::Fit.to_geojson(path) + coords = geojson["features"][0]["geometry"]["coordinates"][0] + raise "expected 2 points" unless coords.size == 2 + expected_lat = 100_000_000 * Maptool::Fit::SEMICIRCLE_TO_DEGREES + expected_lon = 200_000_000 * Maptool::Fit::SEMICIRCLE_TO_DEGREES + raise "bad coordinate: #{coords[0]}" unless (coords[0][1] - expected_lat).abs < 1e-9 && (coords[0][0] - expected_lon).abs < 1e-9 + raise "expected kind track" unless Maptool::Fit.kinds(path) == ["track"] + + empty_path = File.join(dir, "empty.fit") + File.binwrite(empty_path, build_fit([])) + raise "expected no features for a pointless file" unless Maptool::Fit.to_geojson(empty_path)["features"].empty? + raise "expected no kinds for a pointless file" unless Maptool::Fit.kinds(empty_path) == [] + + raise "expected ParseError for garbage" rescue nil + begin + Maptool::Fit.to_geojson(File.join(dir, "nonexistent.fit")) + raise "expected an error reading a missing file" + rescue Errno::ENOENT + # fine — this app only ever calls to_geojson with a path it already + # knows exists + end + + garbage_path = File.join(dir, "garbage.fit") + File.binwrite(garbage_path, "not a fit file at all") + begin + Maptool::Fit.to_geojson(garbage_path) + raise "expected ParseError for garbage input" + rescue Maptool::Fit::ParseError + # expected + end + end + + puts "ok" +end diff --git a/lib/maptool/gpx_store.rb b/lib/maptool/gpx_store.rb index 4539d94..f037541 100644 --- a/lib/maptool/gpx_store.rb +++ b/lib/maptool/gpx_store.rb @@ -1,12 +1,17 @@ require "fileutils" require_relative "gpx" +require_relative "fit" module Maptool - # Filesystem-backed store for uploaded GPX files, organized in + # Filesystem-backed store for uploaded GPX/FIT files, organized in # user-chosen folders (arbitrary nesting). All paths coming from the # outside (folder names, file paths in URLs) are resolved and checked # against the store root before touching the filesystem, so path # traversal (e.g. "../../etc") can't escape it. + # + # A .zip upload (Garmin exports FIT files zipped) isn't handled here — + # the caller (app.rb) unpacks it and calls save() once per extracted + # file, same as it would for several files dropped in at once. class GpxStore class InvalidPath < StandardError; end @@ -16,12 +21,12 @@ module Maptool end # Nested { type:, name:, path:, children: } hash describing every - # folder and .gpx file under the store root. + # folder and .gpx/.fit file under the store root. def tree { "type" => "folder", "name" => "root", "path" => "", "children" => list_children(@root, "") } end - # Saves `io`'s contents as a .gpx file under `folder` (created if + # Saves `io`'s contents as a .gpx/.fit file under `folder` (created if # needed), deriving a safe filename from `filename`. Returns the # stored file's path relative to the store root. def save(folder:, filename:, io:) @@ -91,10 +96,12 @@ module Maptool path == @root || path.start_with?(@root + File::SEPARATOR) end + TRACK_EXTENSIONS = %w[.gpx .fit].freeze + def safe_filename(filename) base = File.basename(filename.to_s) base = "upload.gpx" if base.empty? || base == "." || base == ".." - base = "#{base}.gpx" unless base.downcase.end_with?(".gpx") + base = "#{base}.gpx" unless TRACK_EXTENSIONS.any? { |ext| base.downcase.end_with?(ext) } base end @@ -119,6 +126,8 @@ module Maptool { "type" => "folder", "name" => entry, "path" => rel, "children" => list_children(full, rel) } elsif entry.downcase.end_with?(".gpx") { "type" => "file", "name" => entry, "path" => rel, "kinds" => Gpx.kinds(full) } + elsif entry.downcase.end_with?(".fit") + { "type" => "file", "name" => entry, "path" => rel, "kinds" => Fit.kinds(full) } end end end diff --git a/public/js/gpx.js b/public/js/gpx.js index c7ccfb4..2e9827f 100644 --- a/public/js/gpx.js +++ b/public/js/gpx.js @@ -33,7 +33,7 @@ export class GpxPanelControl { mount(sidebar) { this._container = document.createElement("div"); this._container.className = "sidebar-item"; - this._container.innerHTML = ``; + this._container.innerHTML = ``; this._icon = this._container.querySelector(".sidebar-icon"); this._icon.addEventListener("click", () => this._togglePanel()); @@ -339,7 +339,10 @@ export class GpxPanelControl { } async _uploadFiles(fileList, folder) { - const files = [...fileList].filter((file) => file.name.toLowerCase().endsWith(".gpx")); + // .zip: Garmin's bulk/FIT export format — the server expands it into + // its .gpx/.fit entries, so it's accepted here even though the tree + // never shows a .zip itself. + const files = [...fileList].filter((file) => /\.(gpx|fit|zip)$/i.test(file.name)); if (files.length === 0) return; this._setStatus(files.length === 1 ? "Uploading…" : `Uploading ${files.length} files…`); diff --git a/views/index.erb b/views/index.erb index 864156d..712b889 100644 --- a/views/index.erb +++ b/views/index.erb @@ -36,15 +36,15 @@