From 5ed80ff5b7e2a6fb740dad340be48feed74ee0fa Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Wed, 25 Feb 2026 13:18:40 +0300 Subject: [PATCH 01/18] Add Quantization as a protobuf plugin --- README.md | 198 ++++++++++++++++++ proto/decentraland/common/options.proto | 30 +++ .../common/quantization_example.proto | 132 ++++++++++++ protoc-gen-bitwise/generator_csharp.py | 167 +++++++++++++++ protoc-gen-bitwise/options_pb2.py | 171 +++++++++++++++ protoc-gen-bitwise/plugin.py | 92 ++++++++ protoc-gen-bitwise/runtime/cs/BitReader.cs | 112 ++++++++++ protoc-gen-bitwise/runtime/cs/BitWriter.cs | 117 +++++++++++ protoc-gen-bitwise/runtime/cs/Quantize.cs | 30 +++ 9 files changed, 1049 insertions(+) create mode 100644 proto/decentraland/common/options.proto create mode 100644 proto/decentraland/common/quantization_example.proto create mode 100644 protoc-gen-bitwise/generator_csharp.py create mode 100644 protoc-gen-bitwise/options_pb2.py create mode 100644 protoc-gen-bitwise/plugin.py create mode 100644 protoc-gen-bitwise/runtime/cs/BitReader.cs create mode 100644 protoc-gen-bitwise/runtime/cs/BitWriter.cs create mode 100644 protoc-gen-bitwise/runtime/cs/Quantize.cs diff --git a/README.md b/README.md index 598bdb0b..20c75605 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,201 @@ In this case, there is no problem with when each PR is merged. It's recommendabl ## Comms TODO + +--- + +# Bitwise Serialization Plugin (`protoc-gen-bitwise`) + +A custom protoc plugin that generates C# partial classes with typed float +accessors for quantized `uint32` fields in high-frequency MMO networking +messages (position deltas, player input, etc.). It runs alongside +`--csharp_out` in the same protoc invocation; the two output files coexist +via C# `partial class`. + +## How it works + +Protobuf encodes `uint32` values as varints, which are already compact for +small values: a value up to 2¹⁴−1 costs 2 bytes, up to 2²¹−1 costs 3 bytes. +Rather than a separate binary packing layer, the plugin leverages this: + +1. Declare quantized fields as `uint32` in the `.proto` schema and annotate + them with `[(decentraland.common.quantized)]` to specify the float range + and bit resolution. +2. `--csharp_out` generates the standard protobuf class with the raw `uint32` + property (e.g. `PositionX`). +3. `--bitwise_out` (this plugin) generates a `partial class` extension with a + cached float accessor (e.g. `PositionXQuantized`) that encodes/decodes + transparently via `Quantize.Encode` / `Quantize.Decode`. + +The wire representation is a standard protobuf message — any protobuf-capable +client can read it without knowledge of the plugin. + +## Prerequisites + +| Requirement | Version | +|---|---| +| Python | 3.10+ | +| `protobuf` Python package | 4.x or 3.20+ | +| `protoc` | 3.19+ | + +```bash +pip install protobuf +``` + +## Step 1 — Annotate your `.proto` file + +Declare quantized fields as `uint32` and import `options.proto`: + +```protobuf +syntax = "proto3"; + +import "decentraland/common/options.proto"; + +package decentraland.kernel.comms.v3; + +message PositionDelta { + // Float range [-100, 100] quantized to 16 bits ≈ 0.003-unit precision. + // Stored as uint32 on the wire; protobuf encodes it as a 3-byte varint. + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + + // Unannotated uint32: protobuf varint encodes small values compactly by default. + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; +} +``` + +### Annotation reference + +| Annotation | Target type | Parameters | Effect | +|---|---|---|---| +| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | +| `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | + +### Wire cost at worst-case (all bits set) + +| Quantization bits | Max value | Varint bytes | Tag (field ≤ 15) | Total per field | +|---|---|---|---|---| +| 8 | 255 | 2 | 1 | 3 B | +| 12 | 4 095 | 2 | 1 | 3 B | +| 14 | 16 383 | 2 | 1 | 3 B | +| 16 | 65 535 | 3 | 1 | 4 B | +| 20 | 1 048 575 | 3 | 1 | 4 B | + +Proto3 omits fields equal to their default value (0), so average cost is lower. + +## Step 2 — Run protoc + +```bash +protoc \ + --proto_path=proto \ + --proto_path=/path/to/google/protobuf/include \ + --csharp_out=generated/cs \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.py \ + --bitwise_out=generated/cs \ + proto/decentraland/kernel/comms/v3/comms.proto +``` + +The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output +directory) for each `.proto` file that contains at least one `[(quantized)]` +field. + +## Step 3 — Copy the runtime + +Copy `Quantize.cs` into your project: + +``` +Assets/ +└── Scripts/ + └── Networking/ + └── Bitwise/ + └── Quantize.cs ← protoc-gen-bitwise/runtime/cs/Quantize.cs +``` + +`Quantize.cs` lives in the `Decentraland.Networking.Bitwise` namespace and +provides two static methods used by the generated accessors: + +```csharp +public static class Quantize +{ + public static uint Encode(float value, float min, float max, int bits); + public static float Decode(uint encoded, float min, float max, int bits); +} +``` + +## Step 4 — Use the generated code + +The plugin emits a `partial class` that adds float accessors on top of the +standard protobuf-generated `uint32` properties: + +```csharp +using Decentraland.Kernel.Comms.V3; + +// --- Build and send --- +var delta = new PositionDelta(); +delta.DxQuantized = 3.14f; // encodes to uint32, stored in delta.Dx +delta.DyQuantized = 0f; +delta.DzQuantized = -7.5f; +delta.EntityId = 42u; + +byte[] bytes = delta.ToByteArray(); // standard protobuf serialization +SendOnChannel1(bytes); + +// --- Receive and read --- +var received = PositionDelta.Parser.ParseFrom(receivedBytes); +float x = received.DxQuantized; // decoded on first access, cached thereafter +float y = received.DyQuantized; +float z = received.DzQuantized; + +// If raw uint32 fields are mutated directly after construction, invalidate the cache: +received.ResetDecodedCache(); +``` + +## Generated file example + +For the `PositionDelta` message above the plugin emits `PositionDelta.Bitwise.cs`: + +```csharp +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/kernel/comms/v3/comms.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Kernel.Comms.V3 +{ + public partial class PositionDelta + { + private float? _dx; + public float DxQuantized + { + get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dy; + public float DyQuantized + { + get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dz; + public float DzQuantized + { + get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _dx = null; + _dy = null; + _dz = null; + } + } + +} // namespace Decentraland.Kernel.Comms.V3 +``` diff --git a/proto/decentraland/common/options.proto b/proto/decentraland/common/options.proto new file mode 100644 index 00000000..8e911547 --- /dev/null +++ b/proto/decentraland/common/options.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package decentraland.common; + +import "google/protobuf/descriptor.proto"; + +// Options for quantizing a float value stored as a uint32 field. +// The float is clamped to [min, max] and uniformly quantized to N bits; +// protobuf encodes the resulting uint32 as a varint on the wire. +// The generator emits a float {Name}Quantized accessor in the partial class. +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +// Options for bit-packing an integer field into fewer than 32 bits. +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + // Apply to uint32 fields to enable quantized float encoding. + // Example: uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + QuantizedFloatOptions quantized = 50001; + + // Apply to uint32 fields to pack into fewer bits. + // Example: uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + BitPackedOptions bit_packed = 50002; +} diff --git a/proto/decentraland/common/quantization_example.proto b/proto/decentraland/common/quantization_example.proto new file mode 100644 index 00000000..a82e430c --- /dev/null +++ b/proto/decentraland/common/quantization_example.proto @@ -0,0 +1,132 @@ +syntax = "proto3"; + +package decentraland.common; + +import "decentraland/common/options.proto"; + +// High-frequency player state messages sent on Channel 1 (unreliable sequenced). +// +// Every message below uses the protoc-gen-bitwise annotations to minimise +// wire size. Wire costs are listed per-message so the trade-offs are clear. +// +// Annotation cheat-sheet: +// [(decentraland.common.quantized) = { min: F, max: F, bits: N }] +// → stores a float as a uint32 quantized to N bits over [min, max]. +// → protobuf encodes the uint32 as a varint: values ≤ 2^14-1 cost 2 bytes, +// values ≤ 2^21-1 cost 3 bytes; the generated partial class adds a +// float {Name}Quantized accessor that encodes/decodes transparently. +// [(decentraland.common.bit_packed) = { bits: N }] +// → documents that this uint32 uses at most N bits; protobuf encodes it +// as a varint (same savings, no generated accessor needed). +// (no annotation) +// → standard protobuf encoding (bool/double/etc. at natural width). +// +// Varint byte costs at worst-case (all bits set): +// ≤ 7 bits → 1 byte (max 127) +// ≤ 14 bits → 2 bytes (max 16 383) +// ≤ 21 bits → 3 bytes (max 2 097 151) +// Proto3 omits fields equal to their default value (0), so average cost is lower. + +// --------------------------------------------------------------------------- +// PositionDelta — Δ position relative to last acknowledged full snapshot. +// +// Sent every client tick (~10 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// dx | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dy | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dz | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 19 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: dx/dy/dz ≈ 0.003 units +// --------------------------------------------------------------------------- +message PositionDelta { + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// --------------------------------------------------------------------------- +// PlayerInput — client input snapshot for server-side reconciliation. +// +// Sent every client frame (~30 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// move_x | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// move_z | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// buttons | uint32 | bitmask | 8 | tag 1B + varint 2B = 3B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 15 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: move_x/move_z ≈ 0.008; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message PlayerInput { + // Normalised joystick axes in [-1, 1]. + uint32 move_x = 1 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + uint32 move_z = 2 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + + // Horizontal look direction in degrees. + uint32 yaw = 3 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Bitmask of active buttons (see ButtonFlags below). + uint32 buttons = 4 [(decentraland.common.bit_packed) = { bits: 8 }]; + + // Rolling input sequence number used by the server for reconciliation. + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// Bitmask values for PlayerInput.buttons. +enum ButtonFlags { + BF_NONE = 0; + BF_JUMP = 1; // bit 0 + BF_SPRINT = 2; // bit 1 + BF_INTERACT = 4; // bit 2 + BF_EMOTE = 8; // bit 3 + BF_CROUCH = 16; // bit 4 + // bits 5-7 reserved +} + +// --------------------------------------------------------------------------- +// AvatarStateSnapshot — full authoritative state, sent on Channel 0 (reliable) +// or on resync requests. Demonstrates wider ranges and mixed encodings. +// +// Field | Type | Range | Q bits | Wire worst-case +// -----------------|--------|------------------|--------|----------------------- +// x | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// y | uint32 | [-256, 256] | 14 | tag 1B + varint 2B = 3B +// z | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// pitch | uint32 | [-90, 90] | 10 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// animation_state | uint32 | [0, 63] | 6 | tag 1B + varint 1B = 2B +// is_grounded | bool | — | — | tag 1B + varint 1B = 2B +// timestamp | double | — | — | tag 1B + fixed64 8B = 9B +// --------------------------------------------------------------------------- +// Worst-case: 34 B (vs. 45 B raw: 5×float + 2×uint32 + bool + double) +// Step: x/z ≈ 0.125 units; y ≈ 0.031 units; pitch ≈ 0.176°; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message AvatarStateSnapshot { + // World-space position. + uint32 x = 1 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + uint32 y = 2 [(decentraland.common.quantized) = { min: -256.0, max: 256.0, bits: 14 }]; + uint32 z = 3 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + + // View angles. + uint32 pitch = 4 [(decentraland.common.quantized) = { min: -90.0, max: 90.0, bits: 10 }]; + uint32 yaw = 5 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Identity and animation state. + uint32 entity_id = 6 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 animation_state = 7 [(decentraland.common.bit_packed) = { bits: 6 }]; + + // Un-annotated fields — encoded at their natural width. + bool is_grounded = 8; + double timestamp = 9; // server epoch milliseconds +} diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py new file mode 100644 index 00000000..918c58ad --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.py @@ -0,0 +1,167 @@ +""" +C# code generator for the protoc-gen-bitwise plugin. + +For every proto message that contains at least one uint32 field annotated with +[(quantized)], this module emits a C# partial class that adds a computed float +property named {FieldName}Quantized. The getter decodes the stored uint32 to +a float; the setter encodes a float back to a uint32. Standard protobuf +handles serialization of the uint32 wire field; this class adds a typed float +accessor on top. + +Protobuf encodes small uint32 values via varint, so a value that fits in +2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes). + +Only uint32 fields are supported. bit_packed and unannotated fields are +passed through without generating any accessor. +""" + +from google.protobuf import descriptor_pb2 + +from options_pb2 import get_field_options + +# FieldDescriptorProto type constants (aliased for readability) +_FT = descriptor_pb2.FieldDescriptorProto + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _snake_to_pascal(name: str) -> str: + """position_x → PositionX""" + return ''.join(word.capitalize() for word in name.split('_')) + + +def _package_to_namespace(package: str) -> str: + """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3""" + if not package: + return 'Generated' + return '.'.join(part.capitalize() for part in package.split('.')) + + +def _format_float(value: float) -> str: + """Format a Python float as a C# float literal (e.g. -100.0f).""" + text = f'{value:.8g}' + if '.' not in text and 'e' not in text and 'E' not in text: + text += '.0' + return text + 'f' + + +# --------------------------------------------------------------------------- +# Per-message code generation +# --------------------------------------------------------------------------- + +def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: + """ + Generate a C# partial class for a proto message. + + For each uint32 field with a [(quantized)] annotation, emits a cached float + property {FieldName}Quantized backed by the raw uint32 field. + + Returns a list of lines (without trailing newline) or None if the message + has no quantized uint32 fields. + """ + props: list[tuple[str, str, str, int]] = [] # (prop_name, mn, mx, bits) + + for field in msg_proto.field: + # Repeated/map fields are not supported + if field.label == _FT.LABEL_REPEATED: + continue + + # Only uint32 fields are candidates for quantized accessors + if field.type != _FT.TYPE_UINT32: + continue + + quantized, _ = get_field_options(field.options) + if quantized is None: + continue + + props.append(( + _snake_to_pascal(field.name), + _format_float(quantized.min), + _format_float(quantized.max), + quantized.bits, + )) + + if not props: + return None + + i = indent + backings: list[str] = [] + lines: list[str] = [] + lines.append(f'public partial class {msg_proto.name}') + lines.append('{') + + for prop_name, mn, mx, bits in props: + backing = '_' + prop_name[0].lower() + prop_name[1:] + backings.append(backing) + lines.append(f'{i}private float? {backing};') + lines.append(f'{i}public float {prop_name}Quantized') + lines.append(f'{i}{{') + lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') + lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}') + lines.append(f'{i}}}') + lines.append('') + + lines.append(f'{i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.') + lines.append(f'{i}public void ResetDecodedCache()') + lines.append(f'{i}{{') + for backing in backings: + lines.append(f'{i}{i}{backing} = null;') + lines.append(f'{i}}}') + + lines.append('}') + return lines + + +# --------------------------------------------------------------------------- +# Per-file code generation (public entry point) +# --------------------------------------------------------------------------- + +def generate_csharp(file_proto) -> dict | None: + """ + Generate a C# source file for a FileDescriptorProto. + + Returns a dict with keys 'name' (output path) and 'content' (C# source), + or None if the file contains no quantized uint32 fields. + """ + namespace = _package_to_namespace(file_proto.package) + + # Output is placed flat in the output root, matching --csharp_out convention. + # e.g. "decentraland/kernel/comms/v3/my_message.proto" + # → "MyMessage.Bitwise.cs" + proto_file = file_proto.name.rsplit('/', 1)[-1] + stem = _snake_to_pascal(proto_file.replace('.proto', '')) + out_name = f'{stem}.Bitwise.cs' + + header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + f'// Source: {file_proto.name}', + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + f'namespace {namespace}', + '{', + ] + footer = [ + '', + f'}} // namespace {namespace}', + ] + + body: list[str] = [] + for msg in file_proto.message_type: + msg_lines = _gen_message(msg) + if msg_lines is None: + continue + # Indent each line by 4 spaces (inside the namespace block) + for line in msg_lines: + body.append((' ' + line) if line else '') + body.append('') + + if not body: + return None # nothing to emit + + content = '\n'.join(header + body + footer) + '\n' + return {'name': out_name, 'content': content} diff --git a/protoc-gen-bitwise/options_pb2.py b/protoc-gen-bitwise/options_pb2.py new file mode 100644 index 00000000..d3e637f2 --- /dev/null +++ b/protoc-gen-bitwise/options_pb2.py @@ -0,0 +1,171 @@ +""" +Manual parser for the custom bitwise field options defined in options.proto. + +Rather than relying on protobuf extension registration (which requires a properly +compiled _pb2 module), this module parses the raw serialized FieldOptions bytes +directly using the protobuf binary wire format. All protobuf runtimes preserve +unknown/unregistered extension bytes when round-tripping, so +field.options.SerializeToString() always contains the extension data even when +the extension is not registered in the Python runtime. + +Wire format reference: + tag = (field_number << 3) | wire_type + wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit +""" + +import struct + +# Extension field numbers as defined in options.proto +QUANTIZED_FIELD_NUMBER = 50001 +BIT_PACKED_FIELD_NUMBER = 50002 + + +# --------------------------------------------------------------------------- +# Low-level wire-format helpers +# --------------------------------------------------------------------------- + +def _read_varint(data: bytes, pos: int): + """Decode a protobuf varint starting at *pos*. Returns (value, new_pos).""" + result = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result, pos + + +def _read_float32(data: bytes, pos: int): + """Decode a little-endian 32-bit float. Returns (value, new_pos).""" + value, = struct.unpack_from(' int: + """Advance *pos* past a field with the given wire_type.""" + if wire_type == 0: + _, pos = _read_varint(data, pos) + elif wire_type == 1: + pos += 8 + elif wire_type == 2: + length, pos = _read_varint(data, pos) + pos += length + elif wire_type == 5: + pos += 4 + # wire types 3 and 4 (start/end group) are deprecated; skip gracefully + return pos + + +# --------------------------------------------------------------------------- +# Option message classes +# --------------------------------------------------------------------------- + +class QuantizedFloatOptions: + """Mirrors the QuantizedFloatOptions proto message.""" + + __slots__ = ('min', 'max', 'bits') + + def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0): + self.min = min_val + self.max = max_val + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 5: # min (float) + obj.min, pos = _read_float32(data, pos) + elif field_num == 2 and wire_type == 5: # max (float) + obj.max, pos = _read_float32(data, pos) + elif field_num == 3 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})' + + +class BitPackedOptions: + """Mirrors the BitPackedOptions proto message.""" + + __slots__ = ('bits',) + + def __init__(self, bits: int = 0): + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'BitPackedOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'BitPackedOptions(bits={self.bits})' + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def get_field_options(field_options_proto): + """ + Extract custom bitwise options from a FieldDescriptorProto.options object. + + Serialises the options message to bytes and walks the wire-format stream + looking for extension fields 50001 (quantized) and 50002 (bit_packed). + + Args: + field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance + (may be a default/empty instance when no options are set). + + Returns: + tuple[QuantizedFloatOptions | None, BitPackedOptions | None] + """ + try: + raw = field_options_proto.SerializeToString() + except Exception: + return None, None + + if not raw: + return None, None + + quantized = None + bit_packed = None + pos = 0 + + while pos < len(raw): + tag, pos = _read_varint(raw, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + + if wire_type == 2: # length-delimited + length, pos = _read_varint(raw, pos) + value_bytes = raw[pos:pos + length] + pos += length + if field_num == QUANTIZED_FIELD_NUMBER: + quantized = QuantizedFloatOptions.from_bytes(value_bytes) + elif field_num == BIT_PACKED_FIELD_NUMBER: + bit_packed = BitPackedOptions.from_bytes(value_bytes) + # else: unknown length-delimited field — already consumed + else: + pos = _skip_field(raw, pos, wire_type) + + return quantized, bit_packed diff --git a/protoc-gen-bitwise/plugin.py b/protoc-gen-bitwise/plugin.py new file mode 100644 index 00000000..09b5abe4 --- /dev/null +++ b/protoc-gen-bitwise/plugin.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + +Protocol: + 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + 2. This plugin reads it, generates C# partial classes with Encode / Decode + methods for every message that carries [(quantized)] or [(bit_packed)] + field annotations. + 3. A serialised CodeGeneratorResponse is written to stdout. + +Usage (from project root): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise \\ + proto/my_messages.proto + +Windows invocation (plugin not on PATH as executable): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\ + proto/my_messages.proto + +Dependencies: + pip install grpcio-tools # or: pip install protobuf +""" + +import os +import sys + +# Ensure sibling modules (generator_csharp, options_pb2) are importable +# regardless of where protoc invokes this script from. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# On Windows, stdin/stdout are opened in text mode by default which corrupts +# the binary protobuf payload. +if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + +from google.protobuf.compiler import plugin_pb2 + +from generator_csharp import generate_csharp + + +def main() -> None: + request_bytes = sys.stdin.buffer.read() + + request = plugin_pb2.CodeGeneratorRequest() + request.ParseFromString(request_bytes) + + response = plugin_pb2.CodeGeneratorResponse() + # Advertise proto3-optional support so protoc does not reject the plugin. + response.supported_features = ( + plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL + ) + + # Build a lookup map for all file descriptors (needed for imports, though + # the current generator only uses the directly requested files). + file_by_name = {f.name: f for f in request.proto_file} + + for file_name in request.file_to_generate: + # Skip the options definition file itself — it has no messages to generate. + if file_name == 'decentraland/common/options.proto': + continue + + file_proto = file_by_name.get(file_name) + if file_proto is None: + continue + + try: + generated = generate_csharp(file_proto) + except Exception as exc: # noqa: BLE001 + error = response.file.add() + error.name = '' # empty name signals an error to protoc + # Append error text; protoc will print it and fail. + response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}' + continue + + if generated is not None: + out = response.file.add() + out.name = generated['name'] + out.content = generated['content'] + + sys.stdout.buffer.write(response.SerializeToString()) + + +if __name__ == '__main__': + main() diff --git a/protoc-gen-bitwise/runtime/cs/BitReader.cs b/protoc-gen-bitwise/runtime/cs/BitReader.cs new file mode 100644 index 00000000..51efa39c --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitReader.cs @@ -0,0 +1,112 @@ +// Decentraland.Networking.Bitwise — BitReader +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Reads bits from a byte buffer, MSB first within each byte (big-endian bit + /// order). Symmetric counterpart of : every + /// Write… call has a corresponding Read… call with identical arguments that + /// reproduces the original value. + /// + public sealed class BitReader + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Source buffer filled by a . + public BitReader(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current read position in bits. + public int BitPosition => _bitPos; + + /// + /// Returns true when all written bits have been consumed + /// (i.e. has reached the end of the buffer). + /// + public bool IsAtEnd => _bitPos >= _buffer.Length * 8; + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Reads bits and returns them as the + /// least-significant bits of a , MSB first. + /// + public uint ReadBits(int bits) + { + uint value = 0; + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((_buffer[byteIdx] >> bitIdx & 1) == 1) + value |= 1u << i; + + _bitPos++; + } + return value; + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Reads a quantized float encoded with . + /// Arguments must match those used during encoding exactly. + /// + public float ReadQuantizedFloat(float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + uint quantized = ReadBits(bits); + float normalized = (float)quantized / maxQ; + return min + normalized * (max - min); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers + // ----------------------------------------------------------------- + + /// Reads a 32-bit IEEE 754 float written by . + public float ReadFloat() + { + uint bits = ReadBits(32); + byte[] bytes = + { + (byte)(bits & 0xFF), + (byte)((bits >> 8) & 0xFF), + (byte)((bits >> 16) & 0xFF), + (byte)((bits >> 24) & 0xFF), + }; + return BitConverter.ToSingle(bytes, 0); + } + + /// Reads a 64-bit IEEE 754 double written by . + public double ReadDouble() + { + uint hi = ReadBits(32); + uint lo = ReadBits(32); + byte[] bytes = + { + (byte)(lo & 0xFF), + (byte)((lo >> 8) & 0xFF), + (byte)((lo >> 16) & 0xFF), + (byte)((lo >> 24) & 0xFF), + (byte)(hi & 0xFF), + (byte)((hi >> 8) & 0xFF), + (byte)((hi >> 16) & 0xFF), + (byte)((hi >> 24) & 0xFF), + }; + return BitConverter.ToDouble(bytes, 0); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/BitWriter.cs b/protoc-gen-bitwise/runtime/cs/BitWriter.cs new file mode 100644 index 00000000..19b205b8 --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitWriter.cs @@ -0,0 +1,117 @@ +// Decentraland.Networking.Bitwise — BitWriter +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Writes bits into a pre-allocated byte buffer, MSB first within each byte + /// (big-endian bit order). This matches the layout expected by + /// so that encode → decode is always a round-trip no-op. + /// + public sealed class BitWriter + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Destination buffer (must be large enough for all writes). + public BitWriter(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current write position in bits. + public int BitPosition => _bitPos; + + /// Number of bytes written (rounded up to the nearest byte). + public int ByteLength => (_bitPos + 7) / 8; + + /// Returns a copy of the written bytes (trimmed to ). + public byte[] ToArray() + { + var result = new byte[ByteLength]; + Array.Copy(_buffer, result, ByteLength); + return result; + } + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Writes the least-significant bits of + /// , MSB first. + /// + public void WriteBits(uint value, int bits) + { + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((value >> i & 1u) == 1u) + _buffer[byteIdx] |= (byte)(1 << bitIdx); + else + _buffer[byteIdx] &= (byte)~(1 << bitIdx); + + _bitPos++; + } + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Quantizes into bits + /// using the range [, ] and + /// writes it to the buffer. + /// + /// Uses Math.Round (banker's rounding → ties-to-even) to guarantee + /// that encode → decode is a round-trip no-op. + /// + public void WriteQuantizedFloat(float value, float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + float clamped = Math.Clamp(value, min, max); + float normalized = (clamped - min) / (max - min); + uint quantized = (uint)Math.Round(normalized * maxQ); + WriteBits(quantized, bits); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers (used for un-annotated float/double fields) + // ----------------------------------------------------------------- + + /// Writes a 32-bit IEEE 754 float (4 bytes). + public void WriteFloat(float value) + { + byte[] bytes = BitConverter.GetBytes(value); + // GetBytes is little-endian on all platforms; write MSB first. + uint bits = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + WriteBits(bits, 32); + } + + /// Writes a 64-bit IEEE 754 double (8 bytes). + public void WriteDouble(double value) + { + byte[] bytes = BitConverter.GetBytes(value); + uint lo = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + uint hi = (uint)bytes[4] + | ((uint)bytes[5] << 8) + | ((uint)bytes[6] << 16) + | ((uint)bytes[7] << 24); + // Write high 32 bits first so the bit stream is big-endian at word level too. + WriteBits(hi, 32); + WriteBits(lo, 32); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs new file mode 100644 index 00000000..4ba21a00 --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -0,0 +1,30 @@ +// Decentraland.Networking.Bitwise — Quantize +// Copy this file into your project alongside generated *.Bitwise.cs files. + +namespace Decentraland.Networking.Bitwise; + +/// +/// Static helpers for quantizing float values to/from unsigned integers. +/// Intended for use with protobuf uint32 fields: the integer is transmitted +/// as a protobuf varint, while the float accessor lives in a generated partial class. +/// +public static class Quantize +{ + /// + /// Encodes to a quantized . + /// Values outside [, ] are clamped. + /// + public static uint Encode(float value, float min, float max, int bits) + { + uint steps = (1u << bits) - 1; + float t = Math.Clamp((value - min) / (max - min), 0f, 1f); + return (uint)MathF.Round(t * steps); + } + + /// Decodes a quantized back to a float. + public static float Decode(uint encoded, float min, float max, int bits) + { + uint steps = (1u << bits) - 1; + return (float)encoded / steps * (max - min) + min; + } +} From e51e6633e560c96b605ae8951581b4748738fc72 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Wed, 25 Feb 2026 17:56:10 +0300 Subject: [PATCH 02/18] pulse_comms.proto Signed-off-by: Mikhail Agapov --- proto/decentraland/pulse/pulse_comms.proto | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 proto/decentraland/pulse/pulse_comms.proto diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto new file mode 100644 index 00000000..fb89ca2e --- /dev/null +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package decentraland.pulse; + +message Handshake { + bytes auth_chain = 1; +} + +message ClientMessage { + oneof message { + Handshake handshake = 1; + } +} \ No newline at end of file From 875411948337fd5e0ccf9bbd9520e90d8cec97dc Mon Sep 17 00:00:00 2001 From: Nicolas Lorusso Date: Mon, 2 Mar 2026 10:24:43 -0300 Subject: [PATCH 03/18] add server message & handshake response --- proto/decentraland/pulse/pulse_comms.proto | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index fb89ca2e..234d0938 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -2,12 +2,23 @@ syntax = "proto3"; package decentraland.pulse; -message Handshake { +message HandshakeRequest { bytes auth_chain = 1; } +message HandshakeResponse { + bool success = 1; + optional string error = 2; +} + message ClientMessage { oneof message { - Handshake handshake = 1; + HandshakeRequest handshake = 1; + } +} + +message ServerMessage { + oneof message { + HandshakeResponse handshake = 1; } } \ No newline at end of file From 1e8d37a100c127bdc1ae835e1e471f41ad773e76 Mon Sep 17 00:00:00 2001 From: Nicolas Lorusso Date: Thu, 5 Mar 2026 12:33:06 -0300 Subject: [PATCH 04/18] add player state full/delta --- proto/decentraland/pulse/pulse_comms.proto | 53 ++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index 234d0938..b2ba528e 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -20,5 +20,58 @@ message ClientMessage { message ServerMessage { oneof message { HandshakeResponse handshake = 1; + PlayerStateFull player_state_full = 2; + PlayerStateDelta player_state_delta = 3; } +} + +message PlayerState { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + + Vector3 position = 4; + Vector3 velocity = 5; + + float rotation_y = 6; + + float movement_blend = 7; + float slide_blend = 8; + + float head_yaw = 9; + float head_pitch = 10; + + uint32 state_flags = 11; +} + +message PlayerStateDelta { + uint32 subject_id = 1; + uint32 new_seq = 2; + uint32 server_tick = 3; + + optional uint32 position_x = 4; + optional uint32 position_y = 5; + optional uint32 position_z = 6; + + optional uint32 velocity_x = 7; + optional uint32 velocity_y = 8; + optional uint32 velocity_z = 9; + + optional uint32 rotation_y = 10; + optional uint32 movement_blend = 11; + optional uint32 slide_blend = 12; + optional uint32 head_yaw = 13; + optional uint32 head_pitch = 14; + + uint32 state_flags = 15; +} + +message PlayerStateFull { + PlayerState state = 1; +} + +message Vector3 { + float x = 1; + float y = 2; + float z = 3; } \ No newline at end of file From 5f2a9fe216fbd7d81eed930e37f10a37999836e3 Mon Sep 17 00:00:00 2001 From: Nicolas Lorusso Date: Thu, 5 Mar 2026 15:18:00 -0300 Subject: [PATCH 05/18] add player joined message --- proto/decentraland/pulse/pulse_comms.proto | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index b2ba528e..f6eb342a 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -70,6 +70,11 @@ message PlayerStateFull { PlayerState state = 1; } +message PlayerJoined { + string user_id = 1; + PlayerState state = 2; +} + message Vector3 { float x = 1; float y = 2; From 799c64cf172708752ecbbe0a56ddccfb1d6b9ff2 Mon Sep 17 00:00:00 2001 From: Nicolas Lorusso Date: Thu, 5 Mar 2026 15:28:56 -0300 Subject: [PATCH 06/18] add player joined into server message --- proto/decentraland/pulse/pulse_comms.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index f6eb342a..64bf82e8 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -22,6 +22,7 @@ message ServerMessage { HandshakeResponse handshake = 1; PlayerStateFull player_state_full = 2; PlayerStateDelta player_state_delta = 3; + PlayerJoined player_joined = 4; } } From 26687ced452eb9fb41fa38eb73c82bbefeedaf63 Mon Sep 17 00:00:00 2001 From: Nicolas Lorusso Date: Thu, 5 Mar 2026 15:29:39 -0300 Subject: [PATCH 07/18] add player state into client message --- proto/decentraland/pulse/pulse_comms.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index 64bf82e8..dfffcb53 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -14,6 +14,8 @@ message HandshakeResponse { message ClientMessage { oneof message { HandshakeRequest handshake = 1; + PlayerStateFull player_state_full = 2; + PlayerStateDelta player_state_delta = 3; } } From f597d537994a90b95de3be16a67fab405eedc11c Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Fri, 6 Mar 2026 19:48:46 +0300 Subject: [PATCH 08/18] Update protobufjs to 7.5.4 to resolve the problem with namespaces --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7e731d25..2aeee8cc 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ }, "dependencies": { "@dcl/ts-proto": "1.154.0", - "protobufjs": "7.2.4" + "protobufjs": "7.5.4" }, "files": [ "proto", From c52cda692ccbf33ad9538b66484174c9d6079b50 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Fri, 6 Mar 2026 19:57:13 +0300 Subject: [PATCH 09/18] =?UTF-8?q?proto/google/=20is=20no=20longer=20create?= =?UTF-8?q?d=20or=20published,=20so=20both=20buf=20(CI)=20and=20protoc=20(?= =?UTF-8?q?consumers=20like=20unity-explorer)=20resolve=20=20=20descriptor?= =?UTF-8?q?.proto=20from=20their=20own=20built-in=20includes=20=E2=80=94?= =?UTF-8?q?=20which=20all=20have=20the=20correct=20csharp=5Fnamespace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Mikhail Agapov --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 59cd45af..ee7c1528 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,6 @@ all: buf-lint buf-build test install: npm i - rm -rf proto/google || true - cp -r node_modules/protobufjs/google proto/google list-components-ids: @bash scripts/list-components-ids.sh From 0bed313cfaa8836dcc3e95cd7bcc1c2bbb8ec762 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Fri, 6 Mar 2026 20:14:36 +0300 Subject: [PATCH 10/18] fix: exclude proto/google from npm package to fix C# codegen namespace Revert protobufjs to 7.2.4 (7.5.4 breaks buf and proto-compatibility-tool with newer reserved syntax) and restore the make install copy step for local tooling. Change "files" from "proto" to "proto/decentraland" so the stripped google/protobuf/descriptor.proto (missing csharp_namespace option) is no longer published. Consumers' protoc resolves descriptor.proto from its own built-in includes which have the correct option csharp_namespace = "Google.Protobuf.Reflection". Co-Authored-By: Claude Opus 4.6 --- Makefile | 2 ++ package.json | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ee7c1528..59cd45af 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,8 @@ all: buf-lint buf-build test install: npm i + rm -rf proto/google || true + cp -r node_modules/protobufjs/google proto/google list-components-ids: @bash scripts/list-components-ids.sh diff --git a/package.json b/package.json index 2aeee8cc..59566eb0 100644 --- a/package.json +++ b/package.json @@ -22,10 +22,10 @@ }, "dependencies": { "@dcl/ts-proto": "1.154.0", - "protobufjs": "7.5.4" + "protobufjs": "7.2.4" }, "files": [ - "proto", + "proto/decentraland", "out-ts", "out-js", "public" From 21970a3a7041ae8cbbdc722a529f9f31c9ab63f8 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Fri, 6 Mar 2026 20:48:21 +0300 Subject: [PATCH 11/18] Adjust states encoding --- proto/decentraland/pulse/pulse_comms.proto | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index dfffcb53..6aef1e88 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -28,6 +28,22 @@ message ServerMessage { } } +enum PlayerAnimationFlags { + NONE = 0; + GROUNDED = 1; + LONG_JUMP = 2; + LONG_FALL = 4; + FALLING = 8; + STUNNED = 16; +} + +enum GlideState { + PROP_CLOSED = 0; + OPENING_PROP = 1; + GLIDING = 2; + CLOSING_PROP = 3; +} + message PlayerState { uint32 subject_id = 1; uint32 sequence = 2; @@ -41,10 +57,11 @@ message PlayerState { float movement_blend = 7; float slide_blend = 8; - float head_yaw = 9; - float head_pitch = 10; + optional float head_yaw = 9; + optional float head_pitch = 10; uint32 state_flags = 11; + GlideState glide_state = 12; } message PlayerStateDelta { From 48f751da9fa95ffc06c1698cb96cc4f34c89265f Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Fri, 6 Mar 2026 21:20:07 +0300 Subject: [PATCH 12/18] Add quantization example for PlayerStateDelta Signed-off-by: Mikhail Agapov --- proto/decentraland/pulse/pulse_comms.proto | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index 6aef1e88..17c04339 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -2,6 +2,8 @@ syntax = "proto3"; package decentraland.pulse; +import "decentraland/common/options.proto"; + message HandshakeRequest { bytes auth_chain = 1; } @@ -15,7 +17,6 @@ message ClientMessage { oneof message { HandshakeRequest handshake = 1; PlayerStateFull player_state_full = 2; - PlayerStateDelta player_state_delta = 3; } } @@ -23,7 +24,7 @@ message ServerMessage { oneof message { HandshakeResponse handshake = 1; PlayerStateFull player_state_full = 2; - PlayerStateDelta player_state_delta = 3; + PlayerStateDeltaTier0 player_state_delta = 3; PlayerJoined player_joined = 4; } } @@ -64,7 +65,7 @@ message PlayerState { GlideState glide_state = 12; } -message PlayerStateDelta { +message PlayerStateDeltaTier0 { uint32 subject_id = 1; uint32 new_seq = 2; uint32 server_tick = 3; @@ -77,11 +78,11 @@ message PlayerStateDelta { optional uint32 velocity_y = 8; optional uint32 velocity_z = 9; - optional uint32 rotation_y = 10; + optional uint32 rotation_y = 10 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; optional uint32 movement_blend = 11; optional uint32 slide_blend = 12; - optional uint32 head_yaw = 13; - optional uint32 head_pitch = 14; + optional uint32 head_yaw = 13 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 14 [(decentraland.common.quantized) = { min: 0, max: 180.0, bits: 6 }]; uint32 state_flags = 15; } From 417af3249e391f783121f1eaaeef821f7b1020bd Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Fri, 6 Mar 2026 21:41:23 +0300 Subject: [PATCH 13/18] Copy protoc-gen-bitwise to the output package --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 59566eb0..2022ccf4 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "proto/decentraland", "out-ts", "out-js", - "public" + "public", + "protoc-gen-bitwise" ] } From 2d42c07c750fe231bfcf1e2385b866c2fbf3a923 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Fri, 6 Mar 2026 21:57:22 +0300 Subject: [PATCH 14/18] Make Quantize.cs compitable with Unity Signed-off-by: Mikhail Agapov --- protoc-gen-bitwise/runtime/cs/Quantize.cs | 43 +++++++++++++---------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs index 4ba21a00..22c347fe 100644 --- a/protoc-gen-bitwise/runtime/cs/Quantize.cs +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -1,30 +1,35 @@ // Decentraland.Networking.Bitwise — Quantize // Copy this file into your project alongside generated *.Bitwise.cs files. +// ReSharper disable once RedundantUsingDirective -namespace Decentraland.Networking.Bitwise; +using System; -/// -/// Static helpers for quantizing float values to/from unsigned integers. -/// Intended for use with protobuf uint32 fields: the integer is transmitted -/// as a protobuf varint, while the float accessor lives in a generated partial class. -/// -public static class Quantize +namespace Decentraland.Networking.Bitwise + // ReSharper disable once ArrangeNamespaceBody { /// - /// Encodes to a quantized . - /// Values outside [, ] are clamped. + /// Static helpers for quantizing float values to/from unsigned integers. + /// Intended for use with protobuf uint32 fields: the integer is transmitted + /// as a protobuf varint, while the float accessor lives in a generated partial class. /// - public static uint Encode(float value, float min, float max, int bits) + public static class Quantize { - uint steps = (1u << bits) - 1; - float t = Math.Clamp((value - min) / (max - min), 0f, 1f); - return (uint)MathF.Round(t * steps); - } + /// + /// Encodes to a quantized . + /// Values outside [, ] are clamped. + /// + public static uint Encode(float value, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + var t = Math.Clamp((value - min) / (max - min), 0f, 1f); + return (uint)MathF.Round(t * steps); + } - /// Decodes a quantized back to a float. - public static float Decode(uint encoded, float min, float max, int bits) - { - uint steps = (1u << bits) - 1; - return (float)encoded / steps * (max - min) + min; + /// Decodes a quantized back to a float. + public static float Decode(uint encoded, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + return (float)encoded / steps * (max - min) + min; + } } } From 0888875244c409a9168553437b107c83e718d5c0 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Sun, 8 Mar 2026 19:17:32 +0300 Subject: [PATCH 15/18] Isolate PlayerState as a reusable component Signed-off-by: Mikhail Agapov --- proto/decentraland/pulse/pulse_comms.proto | 43 +++++++++++----------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index 17c04339..cb150c6b 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package decentraland.pulse; import "decentraland/common/options.proto"; +import "decentraland/common/vectors.proto"; message HandshakeRequest { bytes auth_chain = 1; @@ -16,7 +17,7 @@ message HandshakeResponse { message ClientMessage { oneof message { HandshakeRequest handshake = 1; - PlayerStateFull player_state_full = 2; + PlayerStateInput input = 2; } } @@ -45,24 +46,25 @@ enum GlideState { CLOSING_PROP = 3; } -message PlayerState { - uint32 subject_id = 1; - uint32 sequence = 2; - uint32 server_tick = 3; +// Since the server doesn't simulate the scenes state, it trusts the values from the client +message PlayerStateInput { + PlayerState state = 1; +} - Vector3 position = 4; - Vector3 velocity = 5; +message PlayerState { + decentraland.common.Vector3 position = 1; + decentraland.common.Vector3 velocity = 2; - float rotation_y = 6; + float rotation_y = 3; - float movement_blend = 7; - float slide_blend = 8; + float movement_blend = 4; + float slide_blend = 5; - optional float head_yaw = 9; - optional float head_pitch = 10; + optional float head_yaw = 6; + optional float head_pitch = 7; - uint32 state_flags = 11; - GlideState glide_state = 12; + uint32 state_flags = 8; + GlideState glide_state = 9; } message PlayerStateDeltaTier0 { @@ -88,16 +90,13 @@ message PlayerStateDeltaTier0 { } message PlayerStateFull { - PlayerState state = 1; + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; } message PlayerJoined { string user_id = 1; - PlayerState state = 2; -} - -message Vector3 { - float x = 1; - float y = 2; - float z = 3; + PlayerStateFull state = 2; } \ No newline at end of file From a9c8af93355d4ad414d3e1b7cafa7b9cca41c7f2 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Wed, 11 Mar 2026 13:24:46 +0300 Subject: [PATCH 16/18] Add requried quntizationfor delta Signed-off-by: Mikhail Agapov --- proto/decentraland/pulse/pulse_comms.proto | 67 ++++++++++++++-------- protoc-gen-bitwise/generator_csharp.py | 13 ++++- 2 files changed, 53 insertions(+), 27 deletions(-) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index cb150c6b..0cd355f2 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -52,41 +52,52 @@ message PlayerStateInput { } message PlayerState { - decentraland.common.Vector3 position = 1; - decentraland.common.Vector3 velocity = 2; + uint32 parcel_index = 1; + + decentraland.common.Vector3 position = 2; + decentraland.common.Vector3 velocity = 3; - float rotation_y = 3; + float rotation_y = 4; - float movement_blend = 4; - float slide_blend = 5; + float movement_blend = 5; + float slide_blend = 6; - optional float head_yaw = 6; - optional float head_pitch = 7; + optional float head_yaw = 7; + optional float head_pitch = 8; - uint32 state_flags = 8; - GlideState glide_state = 9; + uint32 state_flags = 9; + GlideState glide_state = 10; } message PlayerStateDeltaTier0 { uint32 subject_id = 1; uint32 new_seq = 2; uint32 server_tick = 3; - - optional uint32 position_x = 4; - optional uint32 position_y = 5; - optional uint32 position_z = 6; - - optional uint32 velocity_x = 7; - optional uint32 velocity_y = 8; - optional uint32 velocity_z = 9; - - optional uint32 rotation_y = 10 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; - optional uint32 movement_blend = 11; - optional uint32 slide_blend = 12; - optional uint32 head_yaw = 13 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; - optional uint32 head_pitch = 14 [(decentraland.common.quantized) = { min: 0, max: 180.0, bits: 6 }]; - - uint32 state_flags = 15; + + // While the player doesn't cross the parcel, this field is omitted from diff + optional uint32 parcel_index = 4; + + // X position inside the parcel + optional uint32 position_x = 5 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + // Y position + optional uint32 position_y = 6 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + + // Z position inside the parcel + optional uint32 position_z = 7 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + optional uint32 velocity_x = 8 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_y = 9 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_z = 10 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + + optional uint32 rotation_y = 11 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 movement_blend = 12 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + optional uint32 slide_blend = 13 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + optional uint32 head_yaw = 14 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 15 [(decentraland.common.quantized) = { min: 0, max: 180.0, bits: 6 }]; + + optional uint32 state_flags = 16; + optional GlideState glide_state = 17; } message PlayerStateFull { @@ -96,7 +107,13 @@ message PlayerStateFull { PlayerState state = 4; } +// Notification to the client, that a peer has joined, it can mean connection or entering the area of interest, it's up to the server to decide message PlayerJoined { string user_id = 1; PlayerStateFull state = 2; +} + +// Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest +message PlayerLeft { + uint32 subject_id = 1; } \ No newline at end of file diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py index 918c58ad..376c0dc1 100644 --- a/protoc-gen-bitwise/generator_csharp.py +++ b/protoc-gen-bitwise/generator_csharp.py @@ -47,6 +47,11 @@ def _format_float(value: float) -> str: return text + 'f' +def _format_step(step: float) -> str: + """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003).""" + return f'\u2248 {step:.6g}' + + # --------------------------------------------------------------------------- # Per-message code generation # --------------------------------------------------------------------------- @@ -61,7 +66,7 @@ def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: Returns a list of lines (without trailing newline) or None if the message has no quantized uint32 fields. """ - props: list[tuple[str, str, str, int]] = [] # (prop_name, mn, mx, bits) + props: list[tuple[str, str, str, int, float]] = [] # (prop_name, mn, mx, bits, step) for field in msg_proto.field: # Repeated/map fields are not supported @@ -76,11 +81,14 @@ def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: if quantized is None: continue + step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) + props.append(( _snake_to_pascal(field.name), _format_float(quantized.min), _format_float(quantized.max), quantized.bits, + step, )) if not props: @@ -92,10 +100,11 @@ def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: lines.append(f'public partial class {msg_proto.name}') lines.append('{') - for prop_name, mn, mx, bits in props: + for prop_name, mn, mx, bits, step in props: backing = '_' + prop_name[0].lower() + prop_name[1:] backings.append(backing) lines.append(f'{i}private float? {backing};') + lines.append(f'{i}/// Float accessor for . Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.') lines.append(f'{i}public float {prop_name}Quantized') lines.append(f'{i}{{') lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') From 51f6bdb15fecd059994496e9af09346f33efe8e7 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Wed, 11 Mar 2026 15:38:15 +0300 Subject: [PATCH 17/18] Add RESYNC_REQUEST Signed-off-by: Mikhail Agapov --- proto/decentraland/pulse/pulse_comms.proto | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index 0cd355f2..383ab5c6 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -18,6 +18,7 @@ message ClientMessage { oneof message { HandshakeRequest handshake = 1; PlayerStateInput input = 2; + ResyncRequest resync = 3; } } @@ -27,6 +28,7 @@ message ServerMessage { PlayerStateFull player_state_full = 2; PlayerStateDeltaTier0 player_state_delta = 3; PlayerJoined player_joined = 4; + PlayerLeft player_left = 5; } } @@ -100,6 +102,7 @@ message PlayerStateDeltaTier0 { optional GlideState glide_state = 17; } +// Full State is sent to the client when it is out of sync, and can't recover with a diff only message PlayerStateFull { uint32 subject_id = 1; uint32 sequence = 2; @@ -116,4 +119,12 @@ message PlayerJoined { // Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest message PlayerLeft { uint32 subject_id = 1; +} + +// Client sends resync request if it has a gap in the known sequences and, thus, can't apply a delta. +// It's sent reliably to prevent further desynchronization +message ResyncRequest { + uint32 subject_id = 1; + // highest seq the client actually has for this subject + uint32 known_seq = 2; } \ No newline at end of file From 1526ae84eb5456e5bff0ce6c51049b0f401574a0 Mon Sep 17 00:00:00 2001 From: Mikhail Agapov Date: Fri, 13 Mar 2026 14:52:38 +0300 Subject: [PATCH 18/18] ProfileVersionAnnouncement - parcel_index change uint32 -> int32 Signed-off-by: Mikhail Agapov --- proto/decentraland/pulse/pulse_comms.proto | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto index 383ab5c6..52ada317 100644 --- a/proto/decentraland/pulse/pulse_comms.proto +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -7,6 +7,7 @@ import "decentraland/common/vectors.proto"; message HandshakeRequest { bytes auth_chain = 1; + int32 profile_version = 2; } message HandshakeResponse { @@ -14,11 +15,23 @@ message HandshakeResponse { optional string error = 2; } +// Similarly to the LiveKit pipeline, a peer announces the version of its profile but +// it only does it when it's changed as it's sent reliably and stored on the server for other peers +message ProfileVersionAnnouncement { + int32 version = 1; +} + +message PlayerProfileVersionsAnnounced { + uint32 subject_id = 1; + int32 version = 2; +} + message ClientMessage { oneof message { HandshakeRequest handshake = 1; PlayerStateInput input = 2; ResyncRequest resync = 3; + ProfileVersionAnnouncement profile_announcement = 4; } } @@ -29,6 +42,7 @@ message ServerMessage { PlayerStateDeltaTier0 player_state_delta = 3; PlayerJoined player_joined = 4; PlayerLeft player_left = 5; + PlayerProfileVersionsAnnounced player_profile_version_announced = 6; } } @@ -54,7 +68,7 @@ message PlayerStateInput { } message PlayerState { - uint32 parcel_index = 1; + int32 parcel_index = 1; decentraland.common.Vector3 position = 2; decentraland.common.Vector3 velocity = 3; @@ -77,7 +91,7 @@ message PlayerStateDeltaTier0 { uint32 server_tick = 3; // While the player doesn't cross the parcel, this field is omitted from diff - optional uint32 parcel_index = 4; + optional int32 parcel_index = 4; // X position inside the parcel optional uint32 position_x = 5 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; @@ -113,7 +127,8 @@ message PlayerStateFull { // Notification to the client, that a peer has joined, it can mean connection or entering the area of interest, it's up to the server to decide message PlayerJoined { string user_id = 1; - PlayerStateFull state = 2; + int32 profile_version = 2; + PlayerStateFull state = 3; } // Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest