Support full-width MAVLink source and destination system IDs

.gitignore

@@ -3,6 +3,7 @@
 set_key
 supportproxy
 libraries/
+git-version.h
 venv/
 __pycache__/
 *.pyc

Makefile

@@ -42,16 +42,21 @@ help:
 	@echo "  CXX       - C++ compiler (default: g++)"
 
 # Git submodules
-modules: modules/mavlink/message_definitions/v1.0/all.xml
+modules: modules/mavlink/message_definitions/v1.0/all.xml modules/mavlink/pymavlink/generator/mavgen.py
 
 modules/mavlink/message_definitions/v1.0/all.xml:
 	@echo "Initializing git submodules..."
 	@git submodule update --init --recursive
 
+modules/mavlink/pymavlink/generator/mavgen.py: | modules/mavlink/message_definitions/v1.0/all.xml
+	@git submodule update --init --recursive
+
 # MAVLink headers generation
 headers: $(MAVLINK_DIR)/protocol.h
 
-$(MAVLINK_DIR)/protocol.h: modules/mavlink/message_definitions/v1.0/all.xml
+MAVLINK_INPUTS := $(wildcard modules/mavlink/message_definitions/v1.0/*.xml modules/mavlink/pymavlink/generator/*.py modules/mavlink/pymavlink/generator/C/include_v2.0/*.h)
+
+$(MAVLINK_DIR)/protocol.h: modules/mavlink/message_definitions/v1.0/all.xml regen_headers.sh $(MAVLINK_INPUTS) | modules
 	@echo "Generating MAVLink headers..."
 	@./regen_headers.sh
 
@@ -60,6 +65,9 @@ $(TARGET): $(OBJECTS)
 	@echo "Linking $(TARGET)..."
 	$(CXX) $(CXXFLAGS) -o $@ $^ $(LIBS)
 
+# All users of generated types must rebuild together after a protocol update.
+$(OBJECTS): $(MAVLINK_DIR)/protocol.h
+
 # Object file compilation
 %.o: %.cpp
 	@echo "Compiling $<..."

README.md

@@ -11,13 +11,21 @@ For more information on using the support proxy see https://support.ardupilot.or
 - Both support engineer and user can be on private networks
 - Supports many users running in parallel
 - Uses MAVLink2 signed connections from the support engineer
+- Preserves 32-bit source and destination system IDs across
+  forwarding, signing and telemetry logs
 - Uses normal UDP/TCP forwarding in users GCS
 - Supports both TCP and UDP, including mixed connections
 - Supports WebSocket and WebSocket+SSL TCP connections for both user
   and support engineer
 - supports up to 8 simultaneous connections by support engineer
 - Optional video proxying alongside the MAVLink link, with recording
 
+System IDs above 255 require peers that understand the MAVLink system-ID
+extensions. Older peers cannot read these frames, including proxy diagnostics
+sent with a wide vehicle ID. Before rolling back to an older proxy binary,
+restore configured flight-controller system IDs to the 8-bit range; older
+binaries truncate the binlog source filter, and 256 becomes the match-any value 0.
+
 ## How It Works
 
 ![SupportProxy Architecture](supportproxy-diagram.svg)
@@ -157,8 +165,9 @@ python3 -m venv --system-site-packages venv
 # Activate the virtual environment
 source venv/bin/activate
 
-# Install pymavlink in the virtual environment
-pip install pymavlink
+# Install the pinned pymavlink with 32-bit system ID support
+git submodule update --init --recursive
+pip install ./modules/mavlink/pymavlink
 ```
 
 ## Building SupportProxy
@@ -306,6 +315,7 @@ SupportProxy can also be run using Docker for easier deployment and management.
 ## Building the Docker Image
 
 ```bash
+git submodule update --init --recursive
 docker build -f docker/Dockerfile -t ap-supportproxy .
 ```
 
@@ -390,6 +400,7 @@ The `keydb.py` script provides comprehensive database management:
 ./keydb.py setname PORT2 NewName           # Change name
 ./keydb.py setpass PORT2 NewPassPhrase     # Change passphrase
 ./keydb.py setport1 PORT2 NewPORT1         # Change user port
+./keydb.py setsysid PORT2 SYSID            # FC reboot filter: 1..4294967295; 0 clears
 
 # Reset signing replay-protection timestamp (e.g. after clock skew)
 ./keydb.py resettimestamp PORT2

binlog.h

@@ -112,7 +112,7 @@ class BinlogWriter {
       to packets with msg.sysid == sysid. Sourced from
       KeyEntry.fc_sysid at fork start.
      */
-    void set_fc_sysid_filter(uint8_t sysid) { fc_sysid_filter_ = sysid; }
+    void set_fc_sysid_filter(uint32_t sysid) { fc_sysid_filter_ = sysid; }
 
     /*
       Periodic pump. Called once per main_loop iteration whenever
@@ -230,7 +230,7 @@ class BinlogWriter {
 
     // First-seen sysid/compid of the vehicle on this log session,
     // used as target_{system,component} when we send ACK/NACK back.
-    uint8_t target_system = 0;
+    uint32_t target_system = 0;
     uint8_t target_component = 0;
 
     // Pending ACK queue (seqnos to ACK, FIFO).
@@ -302,7 +302,7 @@ class BinlogWriter {
     // Per-entry MAVLink sysid filter for SYSTEM_TIME-based reboot
     // detection. 0 = match any (default). Set from KeyEntry.fc_sysid
     // by the per-port-pair child at fork.
-    uint8_t fc_sysid_filter_ = 0;
+    uint32_t fc_sysid_filter_ = 0;
 
     // Most-recently-seen SYSTEM_TIME.time_boot_ms from the autopilot.
     // 0 = nothing seen yet (used as a guard so the very first

docker/Dockerfile

@@ -25,14 +25,14 @@ WORKDIR /app
 # Copy source code (from parent directory)
 COPY . .
 
+# The build context excludes Git metadata; initialize submodules on the host.
+RUN test -f modules/mavlink/pymavlink/tools/mavgen.py
+
 # Create virtual environment with system site packages and install pymavlink
 # --system-site-packages allows access to system python3-tdb package
 RUN python3 -m venv --system-site-packages venv && \
     . venv/bin/activate && \
-    pip install --no-cache-dir pymavlink
-
-# Initialize git submodules and build
-RUN git submodule update --init --recursive || true
+    pip install --no-cache-dir ./modules/mavlink/pymavlink
 
 # Build the supportproxy binary with venv activated
 RUN . venv/bin/activate && make clean && make

keydb.py

@@ -264,7 +264,7 @@ def main():
         elif args.action == "setsysid":
             _expect(args.args, 2,
                     "keydb.py setsysid PORT2 SYSID  "
-                    "(0 = match any, 1..255 = filter to that MAVLink sysid)")
+                    "(0 = match any, 1..4294967295 = filter to that MAVLink sysid)")
             try:
                 sysid = int(args.args[1])
             except ValueError:

keydb_lib.py

@@ -796,8 +796,8 @@ def set_fc_sysid(db, port2, sysid):
     ke = KeyEntry(port2)
     if not ke.fetch(db):
         raise CLIError("No entry for port2 %d" % port2)
-    if sysid < 0 or sysid > 255:
-        raise CLIError("fc_sysid must be in 0..255 (got %r)" % sysid)
+    if sysid < 0 or sysid > 0xFFFFFFFF:
+        raise CLIError("fc_sysid must be in 0..4294967295 (got %r)" % sysid)
     ke.fc_sysid = int(sysid)
     ke.store(db)
     return ke

mavlink.cpp

@@ -188,7 +188,7 @@ bool MAVLink::receive_message(uint8_t *&buf, ssize_t &len, mavlink_message_t &ms
 bool MAVLink::send_message(const mavlink_message_t &msg)
 {
     mavlink_message_t msg2 = msg;
-    uint8_t buf[300];
+    uint8_t buf[MAVLINK_MAX_PACKET_LEN];
     if (is_tcp) {
 	if (socket_is_dead(fd)) {
 	    return false;
@@ -235,7 +235,19 @@ bool MAVLink::send_message(const mavlink_message_t &msg)
     // packet loss information
     status->current_tx_seq = msg.seq;
 
-    mavlink_finalize_message_buffer(&msg2, msg2.sysid, msg2.compid, status, min_len, max_len, crc_extra);
+    // Re-sign only the received payload bytes, retaining wide target IDs.
+    uint16_t finalized_len;
+    if (msg.incompat_flags & MAVLINK_IFLAG_TARGET32) {
+        finalized_len = mavlink_finalize_message_buffer_target(
+            &msg2, msg.sysid, msg.compid, status, min_len, msg.len, crc_extra,
+            msg.target_sysid);
+    } else {
+        finalized_len = mavlink_finalize_message_buffer(
+            &msg2, msg.sysid, msg.compid, status, min_len, msg.len, crc_extra);
+    }
+    if (finalized_len == 0) {
+        return false;
+    }
 
     uint16_t len = mavlink_msg_to_send_buffer(buf, &msg2);
     if (len > 0) {
@@ -470,7 +482,7 @@ void MAVLink::mav_printf(uint8_t severity, const char *fmt, ...)
                                      severity,
                                      text,
                                      0, 0);
-    uint8_t buf[300];
+    uint8_t buf[MAVLINK_MAX_PACKET_LEN];
     uint16_t len = mavlink_msg_to_send_buffer(buf, &msg);
     if (len > 0) {
         ::printf("[%d]: %s\n", key_id, text);

mavlink.h

@@ -76,7 +76,8 @@ class MAVLink {
     double last_signing_warning_s = 0;
 
     // last source sysid and compid from a HEARTBEAT from user
-    uint8_t last_sysid, last_compid;
+    uint32_t last_sysid;
+    uint8_t last_compid;
 
     // count of signature errors for triggering message
     uint32_t bad_sig_count = 0;

regen_headers.sh

@@ -1,10 +1,11 @@
 #!/bin/bash
-# re-generate mavlink headers, assumes pymavlink is installed
+# Generate headers with the pymavlink revision pinned by our MAVLink submodule.
+set -euo pipefail
+cd "$(dirname "$0")"
+export PYTHONPATH="$PWD/modules/mavlink${PYTHONPATH:+:$PYTHONPATH}"
 
 echo "Generating mavlink2 headers"
 rm -rf libraries/mavlink2/generated
-mavgen.py --no-validate --wire-protocol 2.0 --lang C modules/mavlink/message_definitions/v1.0/all.xml -o libraries/mavlink2/generated
+python3 modules/mavlink/pymavlink/tools/mavgen.py --no-validate --wire-protocol 2.0 --lang C modules/mavlink/message_definitions/v1.0/all.xml -o libraries/mavlink2/generated
 
 ./git-version.sh
-
-

scripts/run_tests.py

@@ -2,21 +2,21 @@
 """
 Test runner for SupportProxy.
 
-Default behaviour: build supportproxy, then run three pytest invocations
-(connection, authentication, webadmin tests) — each phase has different
+Default behaviour: build supportproxy, then run four pytest invocations
+(connection, authentication, robustness, webadmin tests) — each phase has different
 cwd / keys.tdb / process expectations, so they stay isolated.
 
 Pass -j N for parallel test execution via pytest-xdist; each worker gets
 its own tmpdir and port pair. -j 0 picks one worker per test (so every
 test gets its own worker), useful for the connection phase where the
 slowest worker pins wall-clock time.
 
-Pass --list to enumerate tests across all three phases without running.
+Pass --list to enumerate tests across all four phases without running.
 
 Pass test selectors as positional args (any pytest selector works:
 file path, dir, NodeID, -k expression). When selectors are present the
 runner does ONE pytest invocation against exactly what you asked for,
-skipping the three-phase split.
+skipping the four-phase split.
 """
 import argparse
 import os
@@ -31,7 +31,17 @@
 PHASES = [
     ('Connection Tests',     ['tests/test_connections.py']),
     ('Authentication Tests', ['tests/test_authentication.py']),
-    ('Robustness Tests',     ['tests/test_parent_housekeeping.py',
+    ('Robustness Tests',     ['tests/test_sysid32.py',
+                              'tests/test_binlog_capture.py',
+                              'tests/test_engineer_preauth_pool.py',
+                              'tests/test_engineer_udp_churn.py',
+                              'tests/test_kill_drop.py',
+                              'tests/test_log_cleanup.py',
+                              'tests/test_setup_signing_guard.py',
+                              'tests/test_tlog_capture.py',
+                              'tests/test_run_tests.py',
+                              'tests/test_keydb_log.py',
+                              'tests/test_parent_housekeeping.py',
                               'tests/test_conn2_slot_orphan.py',
                               'tests/test_drop_lost_request.py',
                               'tests/test_websocket_decode.py',
@@ -157,7 +167,7 @@ def build_pytest_cmd(j, extra_args, target_args, timing=False):
 
 
 def cmd_list():
-    """Run pytest --collect-only -q across the three phases."""
+    """Run pytest --collect-only -q across the four phases."""
     for label, targets in PHASES:
         print('\n=== %s ===' % label, flush=True)
         subprocess.call([sys.executable, '-m', 'pytest', '--collect-only',
@@ -175,7 +185,7 @@ def main():
     ap.add_argument('--no-build', action='store_true',
                     help='skip the make step (use existing supportproxy binary)')
     ap.add_argument('--list', action='store_true',
-                    help='list all tests across the three phases and exit')
+                    help='list all tests across the four phases and exit')
     ap.add_argument('--timing', action='store_true',
                     help='print per-test timing at the end, sorted ascending '
                          '(slowest test last)')
@@ -186,7 +196,7 @@ def main():
                          'filter via pytest -k (multiple bare words OR\'d '
                          'together). Mixing both is fine. With selectors '
                          'the runner does one pytest invocation instead '
-                         'of the three default phases.')
+                         'of the four default phases.')
     args = ap.parse_args()
 
     os.chdir(REPO_ROOT)
@@ -206,6 +216,7 @@ def main():
         sys.exit('ERROR: supportproxy binary not found')
 
     all_timings = []
+    failed_phases = []
 
     def run_one(extra_args, target_args):
         cmd = build_pytest_cmd(args.j, extra_args, target_args, args.timing)
@@ -230,20 +241,28 @@ def run_one(extra_args, target_args):
         extra = ['-k', ' or '.join(keywords)] if keywords else []
         if not paths:
             # No path given: search the whole tests/ tree so the keyword
-            # filter applies across all three phases.
+            # filter applies across all four phases.
             paths = ['tests/']
         print('\n=== Running selected tests ===')
         run_one(extra, paths)
     else:
-        # Default: three separate phases (kept apart so phase 2 can wipe
+        # Default: four separate phases (kept apart so phase 2 can wipe
         # keys.tdb without disturbing phase 1's live supportproxy fixture).
         for label, targets in PHASES:
             print('\n=== Running %s ===' % label)
-            run_one([], targets)
+            try:
+                run_one([], targets)
+            except subprocess.CalledProcessError as exc:
+                failed_phases.append(label)
+                print("Phase %s failed with exit code %s" % (label, exc.returncode), flush=True)
 
     if args.timing:
         print_combined_timings(all_timings)
 
+    if failed_phases:
+        print('\nFailed phases: ' + ', '.join(failed_phases))
+        return 1
+
     print('\nAll tests completed.')
     return 0
 

scripts/setup_ci.sh

@@ -41,7 +41,9 @@ source venv/bin/activate
 
 echo "Installing Python dependencies..."
 pip install --upgrade pip
-pip install pytest pytest-xdist pymavlink wsproto Flask Flask-WTF
+git submodule update --init --recursive
+pip install pytest pytest-xdist wsproto Flask Flask-WTF
+pip install ./modules/mavlink/pymavlink
 
 echo "Verifying tdb module accessibility..."
 python3 -c "import tdb; print('tdb module is available')"

supportproxy.cpp

@@ -92,7 +92,7 @@ struct listen_port {
     uint32_t video_flags;
     uint32_t video_flags_hi;   // slots past KEY_VIDEO_PORTS_INLINE
     uint32_t flags;
-    uint8_t  fc_sysid;     // 0 = match any; otherwise the FC's MAVLink
+    uint32_t fc_sysid;     // 0 = match any; otherwise the FC's MAVLink
                            // sysid for binlog reboot detection
     float    tz_offset_hours;  // log-naming timezone (GMT offset in hours)
     bool seen;     // set true by handle_record() during reload_ports()
@@ -182,7 +182,7 @@ static void video_stop_child(struct listen_port *p, const char *why)
     kill(p->video_pid, SIGTERM);
 }
 
-static void upsert_port(int port1, int port2, uint32_t flags, uint8_t fc_sysid,
+static void upsert_port(int port1, int port2, uint32_t flags, uint32_t fc_sysid,
                         float tz_offset_hours, const uint32_t *video_ports,
                         uint32_t video_flags, uint32_t video_flags_hi)
 {
@@ -270,16 +270,13 @@ static int handle_record(struct tdb_context *db, TDB_DATA key, TDB_DATA data, vo
     memcpy(&port2, key.dptr, sizeof(int));
     size_t copy = data.dsize < sizeof(KeyEntry) ? data.dsize : sizeof(KeyEntry);
     memcpy(&k, data.dptr, copy);
-    // KeyEntry.fc_sysid is uint32 for forward compat; the wire value is
-    // a MAVLink sysid (0..255), so truncate to uint8 once it crosses the
-    // C++/binlog boundary. The CLI / web UI already cap at 255.
     // The slots are split across two field groups on disk; hand
     // upsert_port one flat array so nothing downstream has to know.
     uint32_t vports[KEY_MAX_VIDEO_PORTS];
     for (unsigned i = 0; i < KEY_MAX_VIDEO_PORTS; i++) {
         vports[i] = video_port_of(k, i);
     }
-    upsert_port(k.port1, port2, k.flags, uint8_t(k.fc_sysid),
+    upsert_port(k.port1, port2, k.flags, k.fc_sysid,
                 k.tz_offset_hours, vports, k.video_flags, k.video_flags_hi);
     return 0;
 }