Complete desktop streaming, input and audited reliability fixes

.dockerignore

@@ -1,6 +1,13 @@
+# The node image builds from the repository root (deploy/node). Patterns
+# without **/ match only at the root, so every .env (the node's and the
+# relay's hold a Tailscale auth key) is named for any depth: none may reach
+# the build stage, whose layer keeps whatever it copies.
 target
 dist
+dist.noindex
 .git
 **/.DS_Store
-*.app
-.env
+**/*.app
+**/.env
+**/.env.*
+**/__pycache__

.github/workflows/ci.yml

@@ -21,9 +21,9 @@ jobs:
       - name: fmt
         run: cargo fmt --all -- --check
       - name: clippy
-        run: cargo clippy --workspace --all-targets -- -D warnings
+        run: cargo clippy --locked --workspace --all-targets -- -D warnings
       - name: test
-        run: cargo test --workspace
+        run: cargo test --locked --workspace
       # The setup script is generated, so only a dump of it can be parsed.
       # Every plan shape, with names that need quoting, through the parser
       # of the Windows PowerShell that runs it; plus the shipped scripts.
@@ -41,7 +41,7 @@ jobs:
               $env:LATCH_DUMP_SETUP_SCRIPT = $out
               $env:LATCH_DUMP_MIGRATE = $migrate
               $env:LATCH_DUMP_DRY_RUN = $dry
-              cargo test -q -p latch-host --lib the_generated_script_can_be_dumped_for_a_syntax_check
+              cargo test -q --locked -p latch-host --lib the_generated_script_can_be_dumped_for_a_syntax_check
               if ($LASTEXITCODE -ne 0) { throw "dumping the setup script failed" }
               $files += $out
             }
@@ -52,8 +52,104 @@ jobs:
             if ($errors.Count -gt 0) { $errors | Format-List | Out-String | Write-Host; throw "$f does not parse" }
             Write-Host "parses: $f"
           }
+      # The BroLink -> Latch retire step moves the engine folder, with the
+      # pairing state in it, on a PC that ran 4.0. Run it for real here, on
+      # temporary folders and a dummy service, through the Windows
+      # PowerShell setup uses: a clean move, a second run, a folder in the
+      # way (empty and part-filled), and a move that fails.
+      - name: retire step on an old layout
+        shell: powershell
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $root = Join-Path $env:RUNNER_TEMP 'retire'
+          $old = Join-Path $root 'BroLink\engine'
+          $new = Join-Path $root 'Latch\engine'
+          $svc = 'LatchRetireSelftest'
+          $script = Join-Path $env:RUNNER_TEMP 'retire.ps1'
+          $env:LATCH_DUMP_RETIRE_SCRIPT = $script
+          $env:LATCH_DUMP_RETIRE_OLD = $old
+          $env:LATCH_DUMP_RETIRE_NEW = $new
+          $env:LATCH_DUMP_RETIRE_SERVICE = $svc
+          cargo test -q --locked -p latch-host --lib the_retire_step_can_be_dumped_for_a_windows_run
+          if ($LASTEXITCODE -ne 0) { throw "dumping the retire step failed" }
+          function Wait-ServiceGone {
+            foreach ($i in 1..20) {
+              sc.exe query $svc | Out-Null
+              if ($LASTEXITCODE -eq 1060) { return $true }
+              Start-Sleep -Milliseconds 500
+            }
+            return $false
+          }
+          function Reset-Layout {
+            sc.exe query $svc | Out-Null
+            if ($LASTEXITCODE -ne 1060) { sc.exe delete $svc | Out-Null }
+            if (-not (Wait-ServiceGone)) { throw "the dummy service would not go away" }
+            if (Test-Path $root) { Remove-Item $root -Recurse -Force }
+            New-Item -ItemType Directory -Force -Path (Join-Path $old 'config\credentials') | Out-Null
+            Copy-Item "$env:WINDIR\System32\whoami.exe" (Join-Path $old 'sunshine.exe')
+            Set-Content -LiteralPath (Join-Path $old 'config\sunshine_state.json') -Value '{"root":{"uniqueid":"selftest"}}'
+            Set-Content -LiteralPath (Join-Path $old 'config\credentials\cacert.pem') -Value 'cert'
+            Set-Content -LiteralPath (Join-Path $old 'config\credentials\cakey.pem') -Value 'key'
+            sc.exe create $svc binPath= "$env:WINDIR\System32\cmd.exe" start= demand | Out-Null
+            if ($LASTEXITCODE -ne 0) { throw "could not create the dummy service (sc exit $LASTEXITCODE)" }
+          }
+          function Invoke-Retire {
+            $out = & powershell.exe -NoProfile -ExecutionPolicy Bypass -File $script | Out-String
+            Write-Host $out
+            if ($LASTEXITCODE -ne 0) { throw "the retire step exited $LASTEXITCODE" }
+            return $out
+          }
+          function Assert-Moved([string]$why) {
+            foreach ($f in 'config\sunshine_state.json', 'config\credentials\cacert.pem', 'config\credentials\cakey.pem') {
+              if (-not (Test-Path -LiteralPath (Join-Path $new $f))) { throw "${why}: $f did not reach $new" }
+            }
+            if (Test-Path -LiteralPath (Join-Path $new 'engine')) { throw "${why}: the old engine landed inside $new" }
+          }
+
+          Write-Host '== a clean move'
+          Reset-Layout
+          $out = Invoke-Retire
+          Assert-Moved 'clean move'
+          if (Test-Path -LiteralPath (Split-Path $old)) { throw 'clean move: the old folder is still there' }
+          if (-not (Wait-ServiceGone)) { throw 'clean move: the old service was not deleted' }
+          $state = Join-Path $new 'config\sunshine_state.json'
+          $before = (Get-FileHash -LiteralPath $state).Hash
+
+          Write-Host '== a second run changes nothing'
+          $out = Invoke-Retire
+          if ($out -match 'could not') { throw 'second run: it reported a failure' }
+          if ((Get-FileHash -LiteralPath $state).Hash -ne $before) { throw 'second run: the pairing state changed' }
+
+          Write-Host '== an empty engine folder in the way'
+          Reset-Layout
+          New-Item -ItemType Directory -Force -Path $new | Out-Null
+          $out = Invoke-Retire
+          Assert-Moved 'empty folder in the way'
+
+          Write-Host '== a part-filled engine folder in the way'
+          Reset-Layout
+          New-Item -ItemType Directory -Force -Path (Join-Path $new 'tools') | Out-Null
+          Set-Content -LiteralPath (Join-Path $new 'tools\left-over.txt') -Value 'x'
+          $out = Invoke-Retire
+          Assert-Moved 'part-filled folder in the way'
+          if (-not (Get-ChildItem -LiteralPath (Split-Path $new) -Filter 'engine.partial-*')) { throw 'part-filled folder: it was not kept aside' }
+
+          Write-Host '== a move that fails'
+          Reset-Layout
+          # Open without delete sharing, as an antivirus scan or an engine
+          # restarted in the gap would: Windows will not rename the folder.
+          $held = [IO.File]::Open((Join-Path $old 'config\sunshine_state.json'), 'Open', 'Read', 'Read')
+          try { $out = Invoke-Retire } finally { $held.Dispose() }
+          if ($out -notmatch 'could not move the engine \(try 5 of 5\)') { throw 'failed move: it was not retried' }
+          if (-not (Test-Path -LiteralPath (Join-Path $old 'config\sunshine_state.json'))) { throw 'failed move: the old state is gone' }
+          Assert-Moved 'failed move'
+          Write-Host 'retire step: every case passed'
+          # The runner fails a step on the last native command's exit code,
+          # and sc.exe query exits 1060 for a service that is gone, which is
+          # what these checks wait for.
+          exit 0
       - name: release build
-        run: cargo build --workspace --release
+        run: cargo build --locked --workspace --release
       - uses: actions/upload-artifact@v4
         with:
           name: latch-host-windows
@@ -70,16 +166,27 @@ jobs:
           components: clippy
       - uses: Swatinem/rust-cache@v2
       - name: clippy
-        run: cargo clippy --workspace --all-targets -- -D warnings
+        run: cargo clippy --locked --workspace --all-targets -- -D warnings
       - name: test
-        run: cargo test --workspace
+        run: cargo test --locked --workspace
       - name: release app
-        run: cargo build --release -p latch-host --target aarch64-apple-darwin
+        run: cargo build --locked --release -p latch-host --target aarch64-apple-darwin
       - name: bundle
         run: bash scripts/bundle-macos.sh
 
+  # Both production Linux hosts are arm64 (c_char is unsigned there, and
+  # openh264 builds its own assembly); x64 is the other common VPS.
   test-linux:
-    runs-on: ubuntu-latest
+    name: test-linux (${{ matrix.arch }})
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-latest
+            arch: x64
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
@@ -89,11 +196,16 @@ jobs:
       - name: native deps
         run: sudo apt-get update && sudo apt-get install -y --no-install-recommends cmake clang nasm pkg-config libasound2-dev libxkbcommon-dev libwayland-dev libx11-dev libxcursor-dev libxrandr-dev libxi-dev libx11-xcb-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev
       - name: clippy
-        run: cargo clippy --workspace --all-targets -- -D warnings
+        run: cargo clippy --locked --workspace --all-targets -- -D warnings
       - name: test
-        run: cargo test --workspace
+        run: cargo test --locked --workspace
       - name: release host
-        run: cargo build --release -p latch-host
+        run: cargo build --locked --release -p latch-host
+      - uses: actions/upload-artifact@v4
+        with:
+          name: latch-host-linux-${{ matrix.arch }}
+          path: target/release/latch-host
+          if-no-files-found: error
 
   deploy-selftest:
     runs-on: ubuntu-latest

.github/workflows/release.yml

@@ -2,7 +2,8 @@ name: release
 
 # Two ways in:
 #   * workflow_dispatch: build the binaries and leave them on the run page.
-#   * a v* tag: the same binaries, published as a GitHub release.
+#   * a v* tag: the same binaries, published as a GitHub release. The tag has
+#     to be the workspace version (Cargo.toml) with a v in front.
 on:
   workflow_dispatch:
   push:
@@ -28,35 +29,42 @@ jobs:
       - uses: Swatinem/rust-cache@v2
       # Never ship a binary whose tests were not run.
       - name: test
-        run: cargo test --workspace
+        run: cargo test --locked --workspace
       - name: build app
-        run: cargo build --release -p latch-host --target aarch64-apple-darwin
+        run: cargo build --locked --release -p latch-host --target aarch64-apple-darwin
       # bundle-macos.sh ad-hoc signs and verifies the bundle itself.
       - name: bundle
         run: bash scripts/bundle-macos.sh
+      # A Mac refuses an update whose bundle says another version than the
+      # release, deletes the download and fetches it again 15 minutes later.
+      - name: bundle version matches the tag
+        if: startsWith(github.ref, 'refs/tags/v')
+        run: |
+          have="$(/usr/bin/plutil -extract CFBundleShortVersionString raw dist.noindex/Latch.app/Contents/Info.plist)"
+          [ "$have" = "${GITHUB_REF_NAME#v}" ] || { echo "::error::tag ${GITHUB_REF_NAME} but the bundle says ${have}; bump the version in Cargo.toml"; exit 1; }
       - name: package
         run: |
-          cp docs/MACOS.md dist/README-MACOS.md
-          cp LICENSE NOTICE dist/
-          cd dist && tar czf latch-macos-arm64.tar.gz Latch.app README-MACOS.md LICENSE NOTICE
+          cp docs/MACOS.md dist.noindex/README-MACOS.md
+          cp LICENSE NOTICE dist.noindex/
+          cd dist.noindex && tar czf latch-macos-arm64.tar.gz Latch.app README-MACOS.md LICENSE NOTICE
       # A 4.0 updater downloads only its original archive and app names.
       # The same app under that folder name lets it update at all; on its
       # first run it moves itself to Latch.app
       # and takes over the launch agent (crates/host/src/unix_setup.rs,
       # adopt_legacy_bundle). Nothing in 4.1 or later reads this asset.
       - name: package for 4.0 updaters
         run: |
-          mkdir dist/legacy
-          ditto dist/Latch.app dist/legacy/BroLink.app
-          cp docs/MACOS.md dist/legacy/README-MACOS.md
-          cp LICENSE NOTICE dist/legacy/
-          cd dist/legacy && tar czf ../brolink-macos-arm64.tar.gz BroLink.app README-MACOS.md LICENSE NOTICE
+          mkdir dist.noindex/legacy
+          ditto dist.noindex/Latch.app dist.noindex/legacy/BroLink.app
+          cp docs/MACOS.md dist.noindex/legacy/README-MACOS.md
+          cp LICENSE NOTICE dist.noindex/legacy/
+          cd dist.noindex/legacy && tar czf ../brolink-macos-arm64.tar.gz BroLink.app README-MACOS.md LICENSE NOTICE
       - uses: actions/upload-artifact@v4
         with:
           name: latch-macos-arm64
           path: |
-            dist/latch-macos-arm64.tar.gz
-            dist/brolink-macos-arm64.tar.gz
+            dist.noindex/latch-macos-arm64.tar.gz
+            dist.noindex/brolink-macos-arm64.tar.gz
           if-no-files-found: error
 
   windows:
@@ -66,9 +74,9 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
       - name: test
-        run: cargo test --workspace
+        run: cargo test --locked --workspace
       - name: build
-        run: cargo build --release -p latch-host
+        run: cargo build --locked --release -p latch-host
       - name: pin matches setup.rs
         shell: pwsh
         run: |
@@ -120,10 +128,50 @@ jobs:
             brolink-windows-x64.zip
           if-no-files-found: error
 
+  # The Linux host, for VPS desktops: both production ones are arm64, x64 is
+  # the other common kind. Built on Ubuntu 24.04 like CI, so it needs glibc
+  # 2.39 or later. The Mac does not push updates to Linux hosts; they install
+  # from here (deploy/native/README.md) and check the published SHA-256.
+  linux:
+    strategy:
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            arch: x64
+          - runner: ubuntu-24.04-arm
+            arch: arm64
+    runs-on: ${{ matrix.runner }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+      - name: native deps
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends cmake clang nasm pkg-config libasound2-dev libxkbcommon-dev libwayland-dev libx11-dev libxcursor-dev libxrandr-dev libxi-dev libx11-xcb-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev
+      - name: test
+        run: cargo test --locked --workspace
+      - name: build
+        run: cargo build --locked --release -p latch-host
+      - name: package
+        run: |
+          name="latch-host-linux-${{ matrix.arch }}"
+          mkdir -p "dist/$name"
+          cp target/release/latch-host LICENSE NOTICE "dist/$name/"
+          cp -R deploy/native "dist/$name/native"
+          tar -C dist -czf "$name.tar.gz" "$name"
+          sha256sum "$name.tar.gz" > "$name.tar.gz.sha256"
+          cat "$name.tar.gz.sha256"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: latch-host-linux-${{ matrix.arch }}
+          path: |
+            latch-host-linux-${{ matrix.arch }}.tar.gz
+            latch-host-linux-${{ matrix.arch }}.tar.gz.sha256
+          if-no-files-found: error
+
   publish:
     # Tags only: a manual run just leaves artifacts on the run page.
     if: startsWith(github.ref, 'refs/tags/v')
-    needs: [macos, windows]
+    needs: [macos, windows, linux]
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -132,8 +180,17 @@ jobs:
       # before it would be deleted.
       - uses: actions/checkout@v4
         with:
-          sparse-checkout: CHANGELOG.md
+          sparse-checkout: |
+            CHANGELOG.md
+            Cargo.toml
           sparse-checkout-cone-mode: false
+      # The release is named after the tag and every program reports the
+      # workspace version: a tag pushed before the bump would publish a
+      # release no Mac can update to. Read the way bundle-macos.sh reads it.
+      - name: tag matches the workspace version
+        run: |
+          have="$(sed -n 's/^version = "\(.*\)"/\1/p' Cargo.toml | head -n 1)"
+          [ "$have" = "${GITHUB_REF_NAME#v}" ] || { echo "::error::tag ${GITHUB_REF_NAME} but Cargo.toml says ${have}"; exit 1; }
       - uses: actions/download-artifact@v4
         with:
           path: artifacts

.gitignore

@@ -1,5 +1,6 @@
 /target
 /dist
+/dist.noindex
 **/*.rs.bk
 .idea/
 .vscode/