Make call encryption keys work: array shape + room-event transport

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: mx.client
 Type: Package
 Title: Stateful Matrix Client Helpers
-Version: 0.2.1.1
-Date: 2026-10-02
+Version: 0.2.1.3
+Date: 2026-10-07
 Authors@R: c(
     person("Troy", "Hernandez", role = c("aut", "cre"),
            email = "troy@cornball.ai",

NEWS.md

@@ -1,3 +1,42 @@
+# mx.client 0.2.1.3
+
+* Fix: keep every inbound Olm session per peer, not just the latest. A
+  peer can open several Olm sessions to us over its lifetime and reply on
+  any of them; the single inbound slot per peer was overwritten by each
+  new session, so a message sent on a session we had replaced could not be
+  decrypted and was dropped. Found when a FluffyChat MatrixRTC call key
+  arrived on a session the bot had since replaced, leaving the caller's
+  audio undecryptable (`MISSING_KEY`). `olm_in` now holds a list of
+  sessions per peer Curve25519 key; `mx_crypto_process_sync()` appends new
+  inbound sessions instead of replacing, and `olm_receive()` tries them
+  all (plus the outbound session) on receive. Stores written with one
+  session per peer load unchanged (the bare entry becomes a one-element
+  list).
+* The "cannot decrypt Olm to-device message" warning now reports the
+  message type and how many sessions were tried, and includes the
+  `create_inbound` error for a failed prekey, so a dropped message is
+  diagnosable instead of silent.
+
+# mx.client 0.2.1.2
+
+* Fix: make MatrixRTC call encryption keys actually work, found during
+  the first live FluffyChat-to-bot call (the bot heard only undecryptable
+  silence).
+  - `keys` is an array of `{index, key}`, not a single object;
+    `mx_call_key_content()` and `mx_call_key_parse()` treated it as one,
+    so real key events (parsed off the wire with `simplifyVector =
+    FALSE`) were dropped. The parser now reads the array and returns one
+    entry per key.
+  - FluffyChat / matrix-dart-sdk sends the key as an encrypted ROOM
+    event (top-level `device_id`), not to-device. `mx_crypto_process_sync()`
+    now keeps each decrypted room event's `type` and `content`,
+    `mx_call_handle()` reads call keys from `processed$events` as well as
+    `to_device`, and `mx_call_key_parse()` accepts the room-event shape
+    (top-level `device_id`, room taken from the event).
+  - `mx_call_send_key()` also posts the key as an encrypted room event so
+    FluffyChat can decrypt the bot; `mx_send_encrypted()` gained an
+    `event_type` argument.
+
 # mx.client 0.2.1.1
 
 ## MatrixRTC calls

R/call.R

@@ -174,10 +174,18 @@ mx_call_b64_decode <- function(x) {
 # Content of a key event: the key the receiver should use for this
 # device's media. FluffyChat casts every field but sent_ts, so all of them
 # are always present.
+# A superset that both consumers accept: FluffyChat / matrix-dart-sdk
+# reads top-level call_id + device_id + keys (it sends and expects the key
+# as a room event); Element Call reads member + keys (to-device). Sending
+# this one object by either transport reaches both.
 mx_call_key_content <- function(key, index, user_id, device_id, room_id,
                                 now = Sys.time()) {
     list(
-         keys = list(index = as.integer(index), key = mx_call_b64(key)),
+         call_id = "",
+         device_id = device_id,
+         # keys is an array of {index, key}; the wrapping list makes JSON
+         # serialize it as an array even with a single entry.
+         keys = list(list(index = as.integer(index), key = mx_call_b64(key))),
          member = list(id = mx_call_identity(user_id, device_id),
                        claimed_device_id = device_id),
          room_id = room_id,
@@ -186,40 +194,81 @@ mx_call_key_content <- function(key, index, user_id, device_id, room_id,
     )
 }
 
+# Normalize the content's `keys` field to a list of {index, key}. The
+# wire sends an array; depending on how the Olm plaintext was parsed it
+# arrives as an unnamed list of objects (simplifyVector = FALSE), a data
+# frame (simplifyVector = TRUE), and we also tolerate a single {index,
+# key} object from a non-conformant sender.
+mx_call_key_entries <- function(keys) {
+    if (is.null(keys)) {
+        return(list())
+    }
+    if (is.data.frame(keys)) {
+        return(lapply(seq_len(nrow(keys)), function(i) {
+            list(index = keys$index[[i]], key = keys$key[[i]])
+        }))
+    }
+    if (!is.null(names(keys)) && !is.null(keys$index)) {
+        return(list(list(index = keys$index, key = keys$key)))
+    }
+    if (is.list(keys)) {
+        return(Filter(function(e) is.list(e) && !is.null(e$index), keys))
+    }
+    list()
+}
+
 #' Read a call key event
 #'
 #' Parses a decrypted \code{io.element.call.encryption_keys} to-device
 #' event (from the \code{to_device} list of
-#' \code{\link{mx_crypto_process_sync}}) into the LiveKit identity it
-#' belongs to and the key to set for it.
+#' \code{\link{mx_crypto_process_sync}}) into the LiveKit identities and
+#' keys to set. The event's \code{keys} field is an array, so one event
+#' can carry several keys at different indices; each becomes one entry.
 #'
 #' @param event List with \code{type}, \code{content} and \code{sender}.
 #' @param room_id The call's room; keys for other rooms are ignored.
-#' @return \code{list(identity, key, index)}, with \code{key} a raw
-#'   vector, or NULL when the event is not a usable key for this room.
+#' @return A list of \code{list(identity, key, index)}, with \code{key} a
+#'   raw vector, one per usable key in the event; empty when the event is
+#'   not a usable key for this room.
 #' @examples
 #' ev <- list(type = "io.element.call.encryption_keys",
 #'     sender = "@alice:example.org",
-#'     content = list(keys = list(index = 3, key = "AAECAwQFBgcICQoLDA0ODw=="),
+#'     content = list(
+#'         keys = list(list(index = 3, key = "AAECAwQFBgcICQoLDA0ODw==")),
 #'         member = list(claimed_device_id = "PHONE"), room_id = "!r:example.org"))
 #' mx_call_key_parse(ev, "!r:example.org")
 #' @export
 mx_call_key_parse <- function(event, room_id) {
-    if (!identical(event$type, MX_CALL_KEYS)) return(NULL)
+    if (!identical(event$type, MX_CALL_KEYS)) return(list())
     c <- event$content
-    if (!identical(c$room_id, room_id)) return(NULL)
-    device <- c$member$claimed_device_id
-    index <- c$keys$index
-    key <- c$keys$key
+    # A to-device event names its room in the content; a room event does
+    # not (its room is the event's own, matched by the caller). Enforce
+    # content$room_id only when it is present.
+    if (!is.null(c$room_id) && !identical(c$room_id, room_id)) return(list())
+    # Element Call uses member$claimed_device_id; FluffyChat uses a
+    # top-level device_id.
+    device <- c$member$claimed_device_id %||% c$device_id
     if (!is.character(event$sender) || !is.character(device) ||
-        !nzchar(device) || !is.numeric(index) || length(index) != 1L ||
-        index < 0 || index != floor(index) || !is.character(key)) {
-        return(NULL)
+        !nzchar(device)) {
+        return(list())
+    }
+    identity <- mx_call_identity(event$sender, device)
+    out <- list()
+    for (k in mx_call_key_entries(c$keys)) {
+        index <- k$index
+        key <- k$key
+        if (!is.numeric(index) || length(index) != 1L || index < 0 ||
+            index != floor(index) || !is.character(key) || length(key) != 1L) {
+            next
+        }
+        bytes <- tryCatch(mx_call_b64_decode(key), error = function(e) NULL)
+        if (is.null(bytes) || !length(bytes)) {
+            next
+        }
+        out[[length(out) + 1L]] <- list(identity = identity, key = bytes,
+                                        index = as.integer(index))
     }
-    bytes <- tryCatch(mx_call_b64_decode(key), error = function(e) NULL)
-    if (is.null(bytes) || !length(bytes)) return(NULL)
-    list(identity = mx_call_identity(event$sender, device), key = bytes,
-         index = as.integer(index))
+    out
 }
 
 # The key state of one call: our key and index, when the key was made,
@@ -334,6 +383,19 @@ mx_call_send_key <- function(call, targets) {
         }, "")
         call$keys$shared_with <- union(call$keys$shared_with,
                                        intersect(targets, reached))
+        # FluffyChat / matrix-dart-sdk reads the key from an encrypted
+        # ROOM event, not to-device, so also post it there. Guarded: a
+        # failure here must not break the to-device path or the call.
+        tryCatch({
+            user_ids <- unique(sub(":[^:]+$", "", targets))
+            room_res <- mx_send_encrypted(call$client, call$account,
+                call$sessions, call$room_id, content, call$store_dir,
+                member_ids = user_ids, event_type = MX_CALL_KEYS)
+            call$sessions <- room_res$sessions
+        }, error = function(e) {
+            warning("mx.client: could not post the call key as a room event: ",
+                    conditionMessage(e), call. = FALSE)
+        })
     }
     if (!is.null(call$session)) {
         livekitr::lk_set_e2ee_key(call$session, call$keys$key,
@@ -520,9 +582,20 @@ mx_call_handle <- function(call, sync, processed = NULL) {
         call$sessions <- processed$sessions
     }
     received <- character()
-    for (ev in processed$to_device %||% list()) {
-        parsed <- mx_call_key_parse(ev, call$room_id)
-        if (!is.null(parsed)) {
+    # Call keys arrive two ways: Element Call sends them to-device, while
+    # FluffyChat / matrix-dart-sdk sends them as an encrypted ROOM event
+    # (surfaced in processed$events with its type and content). Read both.
+    key_events <- c(processed$to_device %||% list(),
+                    Filter(function(ev) identical(ev$room_id, call$room_id),
+                           processed$events %||% list()))
+    for (ev in key_events) {
+        for (parsed in mx_call_key_parse(ev, call$room_id)) {
+            # Our own key, posted as a room event, echoes back through our
+            # sync; it is not a peer key and must not be applied to our own
+            # participant slot.
+            if (identical(parsed$identity, call$identity)) {
+                next
+            }
             mx_call_apply_peer_key(call, parsed)
             received <- c(received, parsed$identity)
         }

R/e2ee.R

@@ -7,7 +7,12 @@
 #
 # A session set is a list with five named maps:
 #   olm        peer Curve25519 -> outbound Olm session (we encrypt to them)
-#   olm_in     peer Curve25519 -> inbound Olm session  (they encrypt to us)
+#   olm_in     peer Curve25519 -> list of inbound Olm sessions (they encrypt
+#                                 to us). A peer may open several sessions to
+#                                 us over its lifetime and reply on any of
+#                                 them, so we keep every one and try them all
+#                                 on receive; a single slot would drop a
+#                                 message sent on a session we had replaced.
 #   megolm_out room id          -> list(session, shared = peer curves)
 #   megolm_in  "room|session_id" -> list(session, sender, sender_ed25519,
 #                                        sender_bound)
@@ -60,8 +65,8 @@ mx_crypto_sessions_save <- function(sessions, store_dir) {
                  olm = lapply(sessions$olm, function(s) {
         mx.crypto::mxc_olm_session_pickle(s, key)
     }),
-                 olm_in = lapply(sessions$olm_in, function(s) {
-        mx.crypto::mxc_olm_session_pickle(s, key)
+                 olm_in = lapply(sessions$olm_in, function(slist) {
+        lapply(slist, function(s) mx.crypto::mxc_olm_session_pickle(s, key))
     }),
                  megolm_out = lapply(sessions$megolm_out, function(m) {
         list(session = mx.crypto::mxc_megolm_outbound_pickle(m$session, key),
@@ -111,8 +116,13 @@ mx_crypto_sessions_load <- function(store_dir) {
         out$olm[[nm]] <- mx.crypto::mxc_olm_session_unpickle(blob$olm[[nm]], key)
     }
     for (nm in names(blob$olm_in %||% list())) {
-        out$olm_in[[nm]] <- mx.crypto::mxc_olm_session_unpickle(
-            blob$olm_in[[nm]], key)
+        # A store written before olm_in became a list holds a bare pickle
+        # string per peer; wrap it so the in-memory shape is always a list.
+        v <- blob$olm_in[[nm]]
+        pickles <- if (is.list(v)) v else list(v)
+        out$olm_in[[nm]] <- lapply(pickles, function(p) {
+            mx.crypto::mxc_olm_session_unpickle(p, key)
+        })
     }
     for (nm in names(blob$megolm_out %||% list())) {
         m <- blob$megolm_out[[nm]]
@@ -282,7 +292,9 @@ mx_crypto_encrypt_for_devices <- function(account, sessions, room_id,
 #'   \code{is_self} tagging and as the recipient of key requests.
 #' @param self_device_id Character or NULL. This device id. Both this and
 #'   \code{self_id} are required to create room-key requests.
-#' @return List with \code{events} (decrypted, normalized), updated
+#' @return List with \code{events} (decrypted, normalized; each also
+#'   carries its inner \code{type} and \code{content}, so non-message
+#'   room events such as call encryption keys are usable), updated
 #'   \code{verification_events} (original verification envelopes, separated
 #'   from chat messages; no handshake or network side effect is performed),
 #'   \code{sessions}, unsent \code{key_requests}, matching
@@ -343,11 +355,17 @@ mx_crypto_process_sync <- function(account, sessions, sync_resp,
             next
         }
         sender <- ev$content$sender_key
+        # Try every inbound session we hold for this peer, then the outbound
+        # one (Olm sessions are bidirectional). Keeping all of them is what
+        # lets a reply decrypt when the peer picked a session we would
+        # otherwise have overwritten.
         res <- olm_receive(account, sender, msg,
-            list(sessions$olm_in[[sender]], sessions$olm[[sender]]))
+            c(sessions$olm_in[[sender]] %||% list(),
+              list(sessions$olm[[sender]])))
         if (is.null(res)) next
         if (res$new) {
-            sessions$olm_in[[sender]] <- res$session
+            sessions$olm_in[[sender]] <- c(sessions$olm_in[[sender]] %||% list(),
+                                           list(res$session))
         }
         decoded <- olm_decode(res$plaintext)
         if (is.null(decoded)) next
@@ -536,6 +554,12 @@ mx_crypto_process_sync <- function(account, sessions, sync_resp,
                 sender = ev$sender,
                 sender_verified = verified,
                 is_self = isTRUE(ev$sender == self_id),
+                # The inner type and content, kept so non-message room
+                # events (e.g. io.element.call.encryption_keys, which
+                # FluffyChat sends as a room event) are usable by callers
+                # that look past text. Additive; text consumers read body.
+                type = dec$type,
+                content = ct,
                 body = ct$body,
                 msgtype = ct$msgtype,
                 mentions = ct$`m.mentions`$user_ids

R/olm-receive.R

@@ -12,28 +12,41 @@ olm_receive <- function(account, sender, msg, sessions = list()) {
                 call. = FALSE)
         return(NULL)
     }
+    n_tried <- 0L
     for (session in sessions) {
         if (is.null(session)) next
+        n_tried <- n_tried + 1L
         plaintext <- tryCatch(
             mx.crypto::mxc_olm_decrypt(session, msg$type, msg$body),
             error = function(e) NULL)
         if (!is.null(plaintext)) {
             return(list(session = session, plaintext = plaintext, new = FALSE))
         }
     }
+    create_err <- NULL
     if (msg$type == 0L) {
         # A repeated prekey may no longer have an OTK. Do not let that
         # failure abort a sync batch whose earlier messages advanced ratchets.
         result <- tryCatch(mx.crypto::mxc_olm_create_inbound(
             account, peer_curve25519 = sender, prekey_b64 = msg$body),
-            error = function(e) NULL)
+            error = function(e) {
+                create_err <<- conditionMessage(e)
+                NULL
+            })
         if (!is.null(result)) {
             result$new <- TRUE
             return(result)
         }
     }
-    warning("mx.client: cannot decrypt Olm to-device message with ",
-            "available sessions; dropping it", call. = FALSE)
+    # The type and the count of sessions actually tried turn a silent "it
+    # just failed" into something diagnosable: a type-1 reply that no stored
+    # session decrypts means we are missing the session the sender chose
+    # (keep every session per peer and try them all); a type-0 failure
+    # carries the create_inbound error (a spent or malformed one-time key).
+    warning("mx.client: cannot decrypt Olm to-device message (type ",
+            msg$type, ", ", n_tried, " session(s) tried",
+            if (!is.null(create_err)) paste0(", create_inbound: ", create_err)
+            else "", "); dropping it", call. = FALSE)
     NULL
 }
 

R/transport.R

@@ -293,6 +293,8 @@ mx_crypto_verify_claimed_otks <- function(devices, claimed) {
 #'   a room whose other members have no devices is a real room.
 #' @param member_ids Character vector of room member user ids (used when
 #'   \code{recipients} is NULL).
+#' @param event_type Inner Matrix event type, defaulting to
+#'   \code{m.room.message}.
 #' @return List with \code{event_id} and the updated \code{sessions}.
 #' @examples
 #' \dontrun{
@@ -302,7 +304,8 @@ mx_crypto_verify_claimed_otks <- function(devices, claimed) {
 #' }
 #' @export
 mx_send_encrypted <- function(client, account, sessions, room_id, content,
-                              store_dir, recipients = NULL, member_ids = NULL) {
+                              store_dir, recipients = NULL, member_ids = NULL,
+                              event_type = "m.room.message") {
     mx_require_crypto()
     s <- mx_client_session(client)
     sender_curve <- mx.crypto::mxc_account_identity_keys(account)$curve25519
@@ -376,7 +379,7 @@ mx_send_encrypted <- function(client, account, sessions, room_id, content,
 
     out <- mx_crypto_encrypt_for_devices(account, sessions, room_id,
         content, sender_curve, client$device_id, recipients = recipients,
-        sender_user_id = client$user_id)
+        sender_user_id = client$user_id, event_type = event_type)
     for (p in out$to_device) {
         messages <- stats::setNames(
                                     list(stats::setNames(list(p$content), p$device_id)), p$user_id)