diff --git a/README.md b/README.md index 598bdb0b..20c75605 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,201 @@ In this case, there is no problem with when each PR is merged. It's recommendabl ## Comms TODO + +--- + +# Bitwise Serialization Plugin (`protoc-gen-bitwise`) + +A custom protoc plugin that generates C# partial classes with typed float +accessors for quantized `uint32` fields in high-frequency MMO networking +messages (position deltas, player input, etc.). It runs alongside +`--csharp_out` in the same protoc invocation; the two output files coexist +via C# `partial class`. + +## How it works + +Protobuf encodes `uint32` values as varints, which are already compact for +small values: a value up to 2¹⁴−1 costs 2 bytes, up to 2²¹−1 costs 3 bytes. +Rather than a separate binary packing layer, the plugin leverages this: + +1. Declare quantized fields as `uint32` in the `.proto` schema and annotate + them with `[(decentraland.common.quantized)]` to specify the float range + and bit resolution. +2. `--csharp_out` generates the standard protobuf class with the raw `uint32` + property (e.g. `PositionX`). +3. `--bitwise_out` (this plugin) generates a `partial class` extension with a + cached float accessor (e.g. `PositionXQuantized`) that encodes/decodes + transparently via `Quantize.Encode` / `Quantize.Decode`. + +The wire representation is a standard protobuf message — any protobuf-capable +client can read it without knowledge of the plugin. + +## Prerequisites + +| Requirement | Version | +|---|---| +| Python | 3.10+ | +| `protobuf` Python package | 4.x or 3.20+ | +| `protoc` | 3.19+ | + +```bash +pip install protobuf +``` + +## Step 1 — Annotate your `.proto` file + +Declare quantized fields as `uint32` and import `options.proto`: + +```protobuf +syntax = "proto3"; + +import "decentraland/common/options.proto"; + +package decentraland.kernel.comms.v3; + +message PositionDelta { + // Float range [-100, 100] quantized to 16 bits ≈ 0.003-unit precision. + // Stored as uint32 on the wire; protobuf encodes it as a 3-byte varint. + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + + // Unannotated uint32: protobuf varint encodes small values compactly by default. + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; +} +``` + +### Annotation reference + +| Annotation | Target type | Parameters | Effect | +|---|---|---|---| +| `[(decentraland.common.quantized)]` | `uint32` | `min`, `max`, `bits` | Plugin emits a cached `float {Name}Quantized` accessor | +| `[(decentraland.common.bit_packed)]` | `uint32` | `bits` | Documents the value range; protobuf handles varint compaction automatically | + +### Wire cost at worst-case (all bits set) + +| Quantization bits | Max value | Varint bytes | Tag (field ≤ 15) | Total per field | +|---|---|---|---|---| +| 8 | 255 | 2 | 1 | 3 B | +| 12 | 4 095 | 2 | 1 | 3 B | +| 14 | 16 383 | 2 | 1 | 3 B | +| 16 | 65 535 | 3 | 1 | 4 B | +| 20 | 1 048 575 | 3 | 1 | 4 B | + +Proto3 omits fields equal to their default value (0), so average cost is lower. + +## Step 2 — Run protoc + +```bash +protoc \ + --proto_path=proto \ + --proto_path=/path/to/google/protobuf/include \ + --csharp_out=generated/cs \ + --plugin=protoc-gen-bitwise=protoc-gen-bitwise/plugin.py \ + --bitwise_out=generated/cs \ + proto/decentraland/kernel/comms/v3/comms.proto +``` + +The plugin emits one `*.Bitwise.cs` file (PascalCase, flat in the output +directory) for each `.proto` file that contains at least one `[(quantized)]` +field. + +## Step 3 — Copy the runtime + +Copy `Quantize.cs` into your project: + +``` +Assets/ +└── Scripts/ + └── Networking/ + └── Bitwise/ + └── Quantize.cs ← protoc-gen-bitwise/runtime/cs/Quantize.cs +``` + +`Quantize.cs` lives in the `Decentraland.Networking.Bitwise` namespace and +provides two static methods used by the generated accessors: + +```csharp +public static class Quantize +{ + public static uint Encode(float value, float min, float max, int bits); + public static float Decode(uint encoded, float min, float max, int bits); +} +``` + +## Step 4 — Use the generated code + +The plugin emits a `partial class` that adds float accessors on top of the +standard protobuf-generated `uint32` properties: + +```csharp +using Decentraland.Kernel.Comms.V3; + +// --- Build and send --- +var delta = new PositionDelta(); +delta.DxQuantized = 3.14f; // encodes to uint32, stored in delta.Dx +delta.DyQuantized = 0f; +delta.DzQuantized = -7.5f; +delta.EntityId = 42u; + +byte[] bytes = delta.ToByteArray(); // standard protobuf serialization +SendOnChannel1(bytes); + +// --- Receive and read --- +var received = PositionDelta.Parser.ParseFrom(receivedBytes); +float x = received.DxQuantized; // decoded on first access, cached thereafter +float y = received.DyQuantized; +float z = received.DzQuantized; + +// If raw uint32 fields are mutated directly after construction, invalidate the cache: +received.ResetDecodedCache(); +``` + +## Generated file example + +For the `PositionDelta` message above the plugin emits `PositionDelta.Bitwise.cs`: + +```csharp +// +// Generated by protoc-gen-bitwise. DO NOT EDIT. +// Source: decentraland/kernel/comms/v3/comms.proto +// + +using Decentraland.Networking.Bitwise; + +namespace Decentraland.Kernel.Comms.V3 +{ + public partial class PositionDelta + { + private float? _dx; + public float DxQuantized + { + get => _dx ??= Quantize.Decode(Dx, -100.0f, 100.0f, 16); + set { _dx = value; Dx = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dy; + public float DyQuantized + { + get => _dy ??= Quantize.Decode(Dy, -100.0f, 100.0f, 16); + set { _dy = value; Dy = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + private float? _dz; + public float DzQuantized + { + get => _dz ??= Quantize.Decode(Dz, -100.0f, 100.0f, 16); + set { _dz = value; Dz = Quantize.Encode(value, -100.0f, 100.0f, 16); } + } + + /// Clears all cached decoded values. Call after mutating raw uint32 fields directly. + public void ResetDecodedCache() + { + _dx = null; + _dy = null; + _dz = null; + } + } + +} // namespace Decentraland.Kernel.Comms.V3 +``` diff --git a/package.json b/package.json index 7e731d25..2022ccf4 100644 --- a/package.json +++ b/package.json @@ -25,9 +25,10 @@ "protobufjs": "7.2.4" }, "files": [ - "proto", + "proto/decentraland", "out-ts", "out-js", - "public" + "public", + "protoc-gen-bitwise" ] } diff --git a/proto/decentraland/common/options.proto b/proto/decentraland/common/options.proto new file mode 100644 index 00000000..8e911547 --- /dev/null +++ b/proto/decentraland/common/options.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package decentraland.common; + +import "google/protobuf/descriptor.proto"; + +// Options for quantizing a float value stored as a uint32 field. +// The float is clamped to [min, max] and uniformly quantized to N bits; +// protobuf encodes the resulting uint32 as a varint on the wire. +// The generator emits a float {Name}Quantized accessor in the partial class. +message QuantizedFloatOptions { + float min = 1; + float max = 2; + uint32 bits = 3; +} + +// Options for bit-packing an integer field into fewer than 32 bits. +message BitPackedOptions { + uint32 bits = 1; +} + +extend google.protobuf.FieldOptions { + // Apply to uint32 fields to enable quantized float encoding. + // Example: uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + QuantizedFloatOptions quantized = 50001; + + // Apply to uint32 fields to pack into fewer bits. + // Example: uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + BitPackedOptions bit_packed = 50002; +} diff --git a/proto/decentraland/common/quantization_example.proto b/proto/decentraland/common/quantization_example.proto new file mode 100644 index 00000000..a82e430c --- /dev/null +++ b/proto/decentraland/common/quantization_example.proto @@ -0,0 +1,132 @@ +syntax = "proto3"; + +package decentraland.common; + +import "decentraland/common/options.proto"; + +// High-frequency player state messages sent on Channel 1 (unreliable sequenced). +// +// Every message below uses the protoc-gen-bitwise annotations to minimise +// wire size. Wire costs are listed per-message so the trade-offs are clear. +// +// Annotation cheat-sheet: +// [(decentraland.common.quantized) = { min: F, max: F, bits: N }] +// → stores a float as a uint32 quantized to N bits over [min, max]. +// → protobuf encodes the uint32 as a varint: values ≤ 2^14-1 cost 2 bytes, +// values ≤ 2^21-1 cost 3 bytes; the generated partial class adds a +// float {Name}Quantized accessor that encodes/decodes transparently. +// [(decentraland.common.bit_packed) = { bits: N }] +// → documents that this uint32 uses at most N bits; protobuf encodes it +// as a varint (same savings, no generated accessor needed). +// (no annotation) +// → standard protobuf encoding (bool/double/etc. at natural width). +// +// Varint byte costs at worst-case (all bits set): +// ≤ 7 bits → 1 byte (max 127) +// ≤ 14 bits → 2 bytes (max 16 383) +// ≤ 21 bits → 3 bytes (max 2 097 151) +// Proto3 omits fields equal to their default value (0), so average cost is lower. + +// --------------------------------------------------------------------------- +// PositionDelta — Δ position relative to last acknowledged full snapshot. +// +// Sent every client tick (~10 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// dx | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dy | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// dz | uint32 | [-100, 100] | 16 | tag 1B + varint 3B = 4B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 19 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: dx/dy/dz ≈ 0.003 units +// --------------------------------------------------------------------------- +message PositionDelta { + uint32 dx = 1 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dy = 2 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 dz = 3 [(decentraland.common.quantized) = { min: -100.0, max: 100.0, bits: 16 }]; + uint32 entity_id = 4 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// --------------------------------------------------------------------------- +// PlayerInput — client input snapshot for server-side reconciliation. +// +// Sent every client frame (~30 Hz) on Channel 1. +// +// Field | Type | Range | Q bits | Wire worst-case +// ------------|--------|----------------|--------|----------------------- +// move_x | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// move_z | uint32 | [-1, 1] | 8 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// buttons | uint32 | bitmask | 8 | tag 1B + varint 2B = 3B +// sequence | uint32 | [0, 4 095] | 12 | tag 1B + varint 2B = 3B +// --------------------------------------------------------------------------- +// Worst-case: 15 B (vs. 20 B raw: 3×float + 2×uint32) +// Step: move_x/move_z ≈ 0.008; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message PlayerInput { + // Normalised joystick axes in [-1, 1]. + uint32 move_x = 1 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + uint32 move_z = 2 [(decentraland.common.quantized) = { min: -1.0, max: 1.0, bits: 8 }]; + + // Horizontal look direction in degrees. + uint32 yaw = 3 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Bitmask of active buttons (see ButtonFlags below). + uint32 buttons = 4 [(decentraland.common.bit_packed) = { bits: 8 }]; + + // Rolling input sequence number used by the server for reconciliation. + uint32 sequence = 5 [(decentraland.common.bit_packed) = { bits: 12 }]; +} + +// Bitmask values for PlayerInput.buttons. +enum ButtonFlags { + BF_NONE = 0; + BF_JUMP = 1; // bit 0 + BF_SPRINT = 2; // bit 1 + BF_INTERACT = 4; // bit 2 + BF_EMOTE = 8; // bit 3 + BF_CROUCH = 16; // bit 4 + // bits 5-7 reserved +} + +// --------------------------------------------------------------------------- +// AvatarStateSnapshot — full authoritative state, sent on Channel 0 (reliable) +// or on resync requests. Demonstrates wider ranges and mixed encodings. +// +// Field | Type | Range | Q bits | Wire worst-case +// -----------------|--------|------------------|--------|----------------------- +// x | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// y | uint32 | [-256, 256] | 14 | tag 1B + varint 2B = 3B +// z | uint32 | [-4096, 4096] | 16 | tag 1B + varint 3B = 4B +// pitch | uint32 | [-90, 90] | 10 | tag 1B + varint 2B = 3B +// yaw | uint32 | [-180, 180] | 12 | tag 1B + varint 2B = 3B +// entity_id | uint32 | [0, 1 048 575] | 20 | tag 1B + varint 3B = 4B +// animation_state | uint32 | [0, 63] | 6 | tag 1B + varint 1B = 2B +// is_grounded | bool | — | — | tag 1B + varint 1B = 2B +// timestamp | double | — | — | tag 1B + fixed64 8B = 9B +// --------------------------------------------------------------------------- +// Worst-case: 34 B (vs. 45 B raw: 5×float + 2×uint32 + bool + double) +// Step: x/z ≈ 0.125 units; y ≈ 0.031 units; pitch ≈ 0.176°; yaw ≈ 0.088° +// --------------------------------------------------------------------------- +message AvatarStateSnapshot { + // World-space position. + uint32 x = 1 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + uint32 y = 2 [(decentraland.common.quantized) = { min: -256.0, max: 256.0, bits: 14 }]; + uint32 z = 3 [(decentraland.common.quantized) = { min: -4096.0, max: 4096.0, bits: 16 }]; + + // View angles. + uint32 pitch = 4 [(decentraland.common.quantized) = { min: -90.0, max: 90.0, bits: 10 }]; + uint32 yaw = 5 [(decentraland.common.quantized) = { min: -180.0, max: 180.0, bits: 12 }]; + + // Identity and animation state. + uint32 entity_id = 6 [(decentraland.common.bit_packed) = { bits: 20 }]; + uint32 animation_state = 7 [(decentraland.common.bit_packed) = { bits: 6 }]; + + // Un-annotated fields — encoded at their natural width. + bool is_grounded = 8; + double timestamp = 9; // server epoch milliseconds +} diff --git a/proto/decentraland/pulse/pulse_comms.proto b/proto/decentraland/pulse/pulse_comms.proto new file mode 100644 index 00000000..52ada317 --- /dev/null +++ b/proto/decentraland/pulse/pulse_comms.proto @@ -0,0 +1,145 @@ +syntax = "proto3"; + +package decentraland.pulse; + +import "decentraland/common/options.proto"; +import "decentraland/common/vectors.proto"; + +message HandshakeRequest { + bytes auth_chain = 1; + int32 profile_version = 2; +} + +message HandshakeResponse { + bool success = 1; + optional string error = 2; +} + +// Similarly to the LiveKit pipeline, a peer announces the version of its profile but +// it only does it when it's changed as it's sent reliably and stored on the server for other peers +message ProfileVersionAnnouncement { + int32 version = 1; +} + +message PlayerProfileVersionsAnnounced { + uint32 subject_id = 1; + int32 version = 2; +} + +message ClientMessage { + oneof message { + HandshakeRequest handshake = 1; + PlayerStateInput input = 2; + ResyncRequest resync = 3; + ProfileVersionAnnouncement profile_announcement = 4; + } +} + +message ServerMessage { + oneof message { + HandshakeResponse handshake = 1; + PlayerStateFull player_state_full = 2; + PlayerStateDeltaTier0 player_state_delta = 3; + PlayerJoined player_joined = 4; + PlayerLeft player_left = 5; + PlayerProfileVersionsAnnounced player_profile_version_announced = 6; + } +} + +enum PlayerAnimationFlags { + NONE = 0; + GROUNDED = 1; + LONG_JUMP = 2; + LONG_FALL = 4; + FALLING = 8; + STUNNED = 16; +} + +enum GlideState { + PROP_CLOSED = 0; + OPENING_PROP = 1; + GLIDING = 2; + CLOSING_PROP = 3; +} + +// Since the server doesn't simulate the scenes state, it trusts the values from the client +message PlayerStateInput { + PlayerState state = 1; +} + +message PlayerState { + int32 parcel_index = 1; + + decentraland.common.Vector3 position = 2; + decentraland.common.Vector3 velocity = 3; + + float rotation_y = 4; + + float movement_blend = 5; + float slide_blend = 6; + + optional float head_yaw = 7; + optional float head_pitch = 8; + + uint32 state_flags = 9; + GlideState glide_state = 10; +} + +message PlayerStateDeltaTier0 { + uint32 subject_id = 1; + uint32 new_seq = 2; + uint32 server_tick = 3; + + // While the player doesn't cross the parcel, this field is omitted from diff + optional int32 parcel_index = 4; + + // X position inside the parcel + optional uint32 position_x = 5 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + // Y position + optional uint32 position_y = 6 [(decentraland.common.quantized) = { min: 0, max: 200, bits: 13 }]; + + // Z position inside the parcel + optional uint32 position_z = 7 [(decentraland.common.quantized) = { min: 0, max: 16, bits: 8 }]; + + optional uint32 velocity_x = 8 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_y = 9 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + optional uint32 velocity_z = 10 [(decentraland.common.quantized) = { min: -50, max: 50, bits: 8 }]; + + optional uint32 rotation_y = 11 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 movement_blend = 12 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + optional uint32 slide_blend = 13 [(decentraland.common.quantized) = { min: 0, max: 1, bits: 4 }]; + optional uint32 head_yaw = 14 [(decentraland.common.quantized) = { min: 0, max: 360.0, bits: 7 }]; + optional uint32 head_pitch = 15 [(decentraland.common.quantized) = { min: 0, max: 180.0, bits: 6 }]; + + optional uint32 state_flags = 16; + optional GlideState glide_state = 17; +} + +// Full State is sent to the client when it is out of sync, and can't recover with a diff only +message PlayerStateFull { + uint32 subject_id = 1; + uint32 sequence = 2; + uint32 server_tick = 3; + PlayerState state = 4; +} + +// Notification to the client, that a peer has joined, it can mean connection or entering the area of interest, it's up to the server to decide +message PlayerJoined { + string user_id = 1; + int32 profile_version = 2; + PlayerStateFull state = 3; +} + +// Notification to the client, that a peer has left, it can mean disconnection or leaving the area of interest +message PlayerLeft { + uint32 subject_id = 1; +} + +// Client sends resync request if it has a gap in the known sequences and, thus, can't apply a delta. +// It's sent reliably to prevent further desynchronization +message ResyncRequest { + uint32 subject_id = 1; + // highest seq the client actually has for this subject + uint32 known_seq = 2; +} \ No newline at end of file diff --git a/protoc-gen-bitwise/generator_csharp.py b/protoc-gen-bitwise/generator_csharp.py new file mode 100644 index 00000000..376c0dc1 --- /dev/null +++ b/protoc-gen-bitwise/generator_csharp.py @@ -0,0 +1,176 @@ +""" +C# code generator for the protoc-gen-bitwise plugin. + +For every proto message that contains at least one uint32 field annotated with +[(quantized)], this module emits a C# partial class that adds a computed float +property named {FieldName}Quantized. The getter decodes the stored uint32 to +a float; the setter encodes a float back to a uint32. Standard protobuf +handles serialization of the uint32 wire field; this class adds a typed float +accessor on top. + +Protobuf encodes small uint32 values via varint, so a value that fits in +2^20 costs 3 bytes on the wire — cheaper than a raw IEEE 754 float (4 bytes). + +Only uint32 fields are supported. bit_packed and unannotated fields are +passed through without generating any accessor. +""" + +from google.protobuf import descriptor_pb2 + +from options_pb2 import get_field_options + +# FieldDescriptorProto type constants (aliased for readability) +_FT = descriptor_pb2.FieldDescriptorProto + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _snake_to_pascal(name: str) -> str: + """position_x → PositionX""" + return ''.join(word.capitalize() for word in name.split('_')) + + +def _package_to_namespace(package: str) -> str: + """decentraland.kernel.comms.v3 → Decentraland.Kernel.Comms.V3""" + if not package: + return 'Generated' + return '.'.join(part.capitalize() for part in package.split('.')) + + +def _format_float(value: float) -> str: + """Format a Python float as a C# float literal (e.g. -100.0f).""" + text = f'{value:.8g}' + if '.' not in text and 'e' not in text and 'E' not in text: + text += '.0' + return text + 'f' + + +def _format_step(step: float) -> str: + """Format a quantization step size for display in a doc comment (e.g. ≈ 0.003).""" + return f'\u2248 {step:.6g}' + + +# --------------------------------------------------------------------------- +# Per-message code generation +# --------------------------------------------------------------------------- + +def _gen_message(msg_proto, indent: str = ' ') -> list[str] | None: + """ + Generate a C# partial class for a proto message. + + For each uint32 field with a [(quantized)] annotation, emits a cached float + property {FieldName}Quantized backed by the raw uint32 field. + + Returns a list of lines (without trailing newline) or None if the message + has no quantized uint32 fields. + """ + props: list[tuple[str, str, str, int, float]] = [] # (prop_name, mn, mx, bits, step) + + for field in msg_proto.field: + # Repeated/map fields are not supported + if field.label == _FT.LABEL_REPEATED: + continue + + # Only uint32 fields are candidates for quantized accessors + if field.type != _FT.TYPE_UINT32: + continue + + quantized, _ = get_field_options(field.options) + if quantized is None: + continue + + step = (quantized.max - quantized.min) / ((1 << quantized.bits) - 1) + + props.append(( + _snake_to_pascal(field.name), + _format_float(quantized.min), + _format_float(quantized.max), + quantized.bits, + step, + )) + + if not props: + return None + + i = indent + backings: list[str] = [] + lines: list[str] = [] + lines.append(f'public partial class {msg_proto.name}') + lines.append('{') + + for prop_name, mn, mx, bits, step in props: + backing = '_' + prop_name[0].lower() + prop_name[1:] + backings.append(backing) + lines.append(f'{i}private float? {backing};') + lines.append(f'{i}/// Float accessor for . Range [{mn}, {mx}], {bits} bits, step {_format_step(step)}.') + lines.append(f'{i}public float {prop_name}Quantized') + lines.append(f'{i}{{') + lines.append(f'{i}{i}get => {backing} ??= Quantize.Decode({prop_name}, {mn}, {mx}, {bits});') + lines.append(f'{i}{i}set {{ {backing} = value; {prop_name} = Quantize.Encode(value, {mn}, {mx}, {bits}); }}') + lines.append(f'{i}}}') + lines.append('') + + lines.append(f'{i}/// Clears all cached decoded values. Call after mutating raw uint32 fields directly.') + lines.append(f'{i}public void ResetDecodedCache()') + lines.append(f'{i}{{') + for backing in backings: + lines.append(f'{i}{i}{backing} = null;') + lines.append(f'{i}}}') + + lines.append('}') + return lines + + +# --------------------------------------------------------------------------- +# Per-file code generation (public entry point) +# --------------------------------------------------------------------------- + +def generate_csharp(file_proto) -> dict | None: + """ + Generate a C# source file for a FileDescriptorProto. + + Returns a dict with keys 'name' (output path) and 'content' (C# source), + or None if the file contains no quantized uint32 fields. + """ + namespace = _package_to_namespace(file_proto.package) + + # Output is placed flat in the output root, matching --csharp_out convention. + # e.g. "decentraland/kernel/comms/v3/my_message.proto" + # → "MyMessage.Bitwise.cs" + proto_file = file_proto.name.rsplit('/', 1)[-1] + stem = _snake_to_pascal(proto_file.replace('.proto', '')) + out_name = f'{stem}.Bitwise.cs' + + header = [ + '// ', + '// Generated by protoc-gen-bitwise. DO NOT EDIT.', + f'// Source: {file_proto.name}', + '// ', + '', + 'using Decentraland.Networking.Bitwise;', + '', + f'namespace {namespace}', + '{', + ] + footer = [ + '', + f'}} // namespace {namespace}', + ] + + body: list[str] = [] + for msg in file_proto.message_type: + msg_lines = _gen_message(msg) + if msg_lines is None: + continue + # Indent each line by 4 spaces (inside the namespace block) + for line in msg_lines: + body.append((' ' + line) if line else '') + body.append('') + + if not body: + return None # nothing to emit + + content = '\n'.join(header + body + footer) + '\n' + return {'name': out_name, 'content': content} diff --git a/protoc-gen-bitwise/options_pb2.py b/protoc-gen-bitwise/options_pb2.py new file mode 100644 index 00000000..d3e637f2 --- /dev/null +++ b/protoc-gen-bitwise/options_pb2.py @@ -0,0 +1,171 @@ +""" +Manual parser for the custom bitwise field options defined in options.proto. + +Rather than relying on protobuf extension registration (which requires a properly +compiled _pb2 module), this module parses the raw serialized FieldOptions bytes +directly using the protobuf binary wire format. All protobuf runtimes preserve +unknown/unregistered extension bytes when round-tripping, so +field.options.SerializeToString() always contains the extension data even when +the extension is not registered in the Python runtime. + +Wire format reference: + tag = (field_number << 3) | wire_type + wire_type 0 = varint, 1 = 64-bit, 2 = length-delimited, 5 = 32-bit +""" + +import struct + +# Extension field numbers as defined in options.proto +QUANTIZED_FIELD_NUMBER = 50001 +BIT_PACKED_FIELD_NUMBER = 50002 + + +# --------------------------------------------------------------------------- +# Low-level wire-format helpers +# --------------------------------------------------------------------------- + +def _read_varint(data: bytes, pos: int): + """Decode a protobuf varint starting at *pos*. Returns (value, new_pos).""" + result = 0 + shift = 0 + while pos < len(data): + byte = data[pos] + pos += 1 + result |= (byte & 0x7F) << shift + if not (byte & 0x80): + break + shift += 7 + return result, pos + + +def _read_float32(data: bytes, pos: int): + """Decode a little-endian 32-bit float. Returns (value, new_pos).""" + value, = struct.unpack_from(' int: + """Advance *pos* past a field with the given wire_type.""" + if wire_type == 0: + _, pos = _read_varint(data, pos) + elif wire_type == 1: + pos += 8 + elif wire_type == 2: + length, pos = _read_varint(data, pos) + pos += length + elif wire_type == 5: + pos += 4 + # wire types 3 and 4 (start/end group) are deprecated; skip gracefully + return pos + + +# --------------------------------------------------------------------------- +# Option message classes +# --------------------------------------------------------------------------- + +class QuantizedFloatOptions: + """Mirrors the QuantizedFloatOptions proto message.""" + + __slots__ = ('min', 'max', 'bits') + + def __init__(self, min_val: float = 0.0, max_val: float = 0.0, bits: int = 0): + self.min = min_val + self.max = max_val + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'QuantizedFloatOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 5: # min (float) + obj.min, pos = _read_float32(data, pos) + elif field_num == 2 and wire_type == 5: # max (float) + obj.max, pos = _read_float32(data, pos) + elif field_num == 3 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'QuantizedFloatOptions(min={self.min}, max={self.max}, bits={self.bits})' + + +class BitPackedOptions: + """Mirrors the BitPackedOptions proto message.""" + + __slots__ = ('bits',) + + def __init__(self, bits: int = 0): + self.bits = bits + + @classmethod + def from_bytes(cls, data: bytes) -> 'BitPackedOptions': + obj = cls() + pos = 0 + while pos < len(data): + tag, pos = _read_varint(data, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + if field_num == 1 and wire_type == 0: # bits (uint32) + obj.bits, pos = _read_varint(data, pos) + else: + pos = _skip_field(data, pos, wire_type) + return obj + + def __repr__(self): + return f'BitPackedOptions(bits={self.bits})' + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + +def get_field_options(field_options_proto): + """ + Extract custom bitwise options from a FieldDescriptorProto.options object. + + Serialises the options message to bytes and walks the wire-format stream + looking for extension fields 50001 (quantized) and 50002 (bit_packed). + + Args: + field_options_proto: google.protobuf.descriptor_pb2.FieldOptions instance + (may be a default/empty instance when no options are set). + + Returns: + tuple[QuantizedFloatOptions | None, BitPackedOptions | None] + """ + try: + raw = field_options_proto.SerializeToString() + except Exception: + return None, None + + if not raw: + return None, None + + quantized = None + bit_packed = None + pos = 0 + + while pos < len(raw): + tag, pos = _read_varint(raw, pos) + field_num = tag >> 3 + wire_type = tag & 0x7 + + if wire_type == 2: # length-delimited + length, pos = _read_varint(raw, pos) + value_bytes = raw[pos:pos + length] + pos += length + if field_num == QUANTIZED_FIELD_NUMBER: + quantized = QuantizedFloatOptions.from_bytes(value_bytes) + elif field_num == BIT_PACKED_FIELD_NUMBER: + bit_packed = BitPackedOptions.from_bytes(value_bytes) + # else: unknown length-delimited field — already consumed + else: + pos = _skip_field(raw, pos, wire_type) + + return quantized, bit_packed diff --git a/protoc-gen-bitwise/plugin.py b/protoc-gen-bitwise/plugin.py new file mode 100644 index 00000000..09b5abe4 --- /dev/null +++ b/protoc-gen-bitwise/plugin.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +""" +protoc-gen-bitwise — protoc plugin that generates C# bitwise serialization code. + +Protocol: + 1. protoc writes a serialised CodeGeneratorRequest to this process's stdin. + 2. This plugin reads it, generates C# partial classes with Encode / Decode + methods for every message that carries [(quantized)] or [(bit_packed)] + field annotations. + 3. A serialised CodeGeneratorResponse is written to stdout. + +Usage (from project root): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise \\ + proto/my_messages.proto + +Windows invocation (plugin not on PATH as executable): + protoc \\ + --proto_path=proto \\ + --bitwise_out=generated/ \\ + --plugin=protoc-gen-bitwise=python protoc-gen-bitwise/plugin.py \\ + proto/my_messages.proto + +Dependencies: + pip install grpcio-tools # or: pip install protobuf +""" + +import os +import sys + +# Ensure sibling modules (generator_csharp, options_pb2) are importable +# regardless of where protoc invokes this script from. +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# On Windows, stdin/stdout are opened in text mode by default which corrupts +# the binary protobuf payload. +if sys.platform == 'win32': + import msvcrt + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + +from google.protobuf.compiler import plugin_pb2 + +from generator_csharp import generate_csharp + + +def main() -> None: + request_bytes = sys.stdin.buffer.read() + + request = plugin_pb2.CodeGeneratorRequest() + request.ParseFromString(request_bytes) + + response = plugin_pb2.CodeGeneratorResponse() + # Advertise proto3-optional support so protoc does not reject the plugin. + response.supported_features = ( + plugin_pb2.CodeGeneratorResponse.FEATURE_PROTO3_OPTIONAL + ) + + # Build a lookup map for all file descriptors (needed for imports, though + # the current generator only uses the directly requested files). + file_by_name = {f.name: f for f in request.proto_file} + + for file_name in request.file_to_generate: + # Skip the options definition file itself — it has no messages to generate. + if file_name == 'decentraland/common/options.proto': + continue + + file_proto = file_by_name.get(file_name) + if file_proto is None: + continue + + try: + generated = generate_csharp(file_proto) + except Exception as exc: # noqa: BLE001 + error = response.file.add() + error.name = '' # empty name signals an error to protoc + # Append error text; protoc will print it and fail. + response.error = f'protoc-gen-bitwise: error processing {file_name}: {exc}' + continue + + if generated is not None: + out = response.file.add() + out.name = generated['name'] + out.content = generated['content'] + + sys.stdout.buffer.write(response.SerializeToString()) + + +if __name__ == '__main__': + main() diff --git a/protoc-gen-bitwise/runtime/cs/BitReader.cs b/protoc-gen-bitwise/runtime/cs/BitReader.cs new file mode 100644 index 00000000..51efa39c --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitReader.cs @@ -0,0 +1,112 @@ +// Decentraland.Networking.Bitwise — BitReader +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Reads bits from a byte buffer, MSB first within each byte (big-endian bit + /// order). Symmetric counterpart of : every + /// Write… call has a corresponding Read… call with identical arguments that + /// reproduces the original value. + /// + public sealed class BitReader + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Source buffer filled by a . + public BitReader(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current read position in bits. + public int BitPosition => _bitPos; + + /// + /// Returns true when all written bits have been consumed + /// (i.e. has reached the end of the buffer). + /// + public bool IsAtEnd => _bitPos >= _buffer.Length * 8; + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Reads bits and returns them as the + /// least-significant bits of a , MSB first. + /// + public uint ReadBits(int bits) + { + uint value = 0; + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((_buffer[byteIdx] >> bitIdx & 1) == 1) + value |= 1u << i; + + _bitPos++; + } + return value; + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Reads a quantized float encoded with . + /// Arguments must match those used during encoding exactly. + /// + public float ReadQuantizedFloat(float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + uint quantized = ReadBits(bits); + float normalized = (float)quantized / maxQ; + return min + normalized * (max - min); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers + // ----------------------------------------------------------------- + + /// Reads a 32-bit IEEE 754 float written by . + public float ReadFloat() + { + uint bits = ReadBits(32); + byte[] bytes = + { + (byte)(bits & 0xFF), + (byte)((bits >> 8) & 0xFF), + (byte)((bits >> 16) & 0xFF), + (byte)((bits >> 24) & 0xFF), + }; + return BitConverter.ToSingle(bytes, 0); + } + + /// Reads a 64-bit IEEE 754 double written by . + public double ReadDouble() + { + uint hi = ReadBits(32); + uint lo = ReadBits(32); + byte[] bytes = + { + (byte)(lo & 0xFF), + (byte)((lo >> 8) & 0xFF), + (byte)((lo >> 16) & 0xFF), + (byte)((lo >> 24) & 0xFF), + (byte)(hi & 0xFF), + (byte)((hi >> 8) & 0xFF), + (byte)((hi >> 16) & 0xFF), + (byte)((hi >> 24) & 0xFF), + }; + return BitConverter.ToDouble(bytes, 0); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/BitWriter.cs b/protoc-gen-bitwise/runtime/cs/BitWriter.cs new file mode 100644 index 00000000..19b205b8 --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/BitWriter.cs @@ -0,0 +1,117 @@ +// Decentraland.Networking.Bitwise — BitWriter +// Copy this file into your Unity project alongside generated *.Bitwise.cs files. + +using System; + +namespace Decentraland.Networking.Bitwise +{ + /// + /// Writes bits into a pre-allocated byte buffer, MSB first within each byte + /// (big-endian bit order). This matches the layout expected by + /// so that encode → decode is always a round-trip no-op. + /// + public sealed class BitWriter + { + private readonly byte[] _buffer; + private int _bitPos; + + /// Destination buffer (must be large enough for all writes). + public BitWriter(byte[] buffer) + { + _buffer = buffer ?? throw new ArgumentNullException(nameof(buffer)); + _bitPos = 0; + } + + /// Current write position in bits. + public int BitPosition => _bitPos; + + /// Number of bytes written (rounded up to the nearest byte). + public int ByteLength => (_bitPos + 7) / 8; + + /// Returns a copy of the written bytes (trimmed to ). + public byte[] ToArray() + { + var result = new byte[ByteLength]; + Array.Copy(_buffer, result, ByteLength); + return result; + } + + // ----------------------------------------------------------------- + // Core primitive + // ----------------------------------------------------------------- + + /// + /// Writes the least-significant bits of + /// , MSB first. + /// + public void WriteBits(uint value, int bits) + { + for (int i = bits - 1; i >= 0; i--) + { + int byteIdx = _bitPos / 8; + int bitIdx = 7 - (_bitPos % 8); + + if ((value >> i & 1u) == 1u) + _buffer[byteIdx] |= (byte)(1 << bitIdx); + else + _buffer[byteIdx] &= (byte)~(1 << bitIdx); + + _bitPos++; + } + } + + // ----------------------------------------------------------------- + // Quantized float + // ----------------------------------------------------------------- + + /// + /// Quantizes into bits + /// using the range [, ] and + /// writes it to the buffer. + /// + /// Uses Math.Round (banker's rounding → ties-to-even) to guarantee + /// that encode → decode is a round-trip no-op. + /// + public void WriteQuantizedFloat(float value, float min, float max, int bits) + { + uint maxQ = (1u << bits) - 1; + float clamped = Math.Clamp(value, min, max); + float normalized = (clamped - min) / (max - min); + uint quantized = (uint)Math.Round(normalized * maxQ); + WriteBits(quantized, bits); + } + + // ----------------------------------------------------------------- + // Standard IEEE 754 helpers (used for un-annotated float/double fields) + // ----------------------------------------------------------------- + + /// Writes a 32-bit IEEE 754 float (4 bytes). + public void WriteFloat(float value) + { + byte[] bytes = BitConverter.GetBytes(value); + // GetBytes is little-endian on all platforms; write MSB first. + uint bits = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + WriteBits(bits, 32); + } + + /// Writes a 64-bit IEEE 754 double (8 bytes). + public void WriteDouble(double value) + { + byte[] bytes = BitConverter.GetBytes(value); + uint lo = (uint)bytes[0] + | ((uint)bytes[1] << 8) + | ((uint)bytes[2] << 16) + | ((uint)bytes[3] << 24); + uint hi = (uint)bytes[4] + | ((uint)bytes[5] << 8) + | ((uint)bytes[6] << 16) + | ((uint)bytes[7] << 24); + // Write high 32 bits first so the bit stream is big-endian at word level too. + WriteBits(hi, 32); + WriteBits(lo, 32); + } + } +} diff --git a/protoc-gen-bitwise/runtime/cs/Quantize.cs b/protoc-gen-bitwise/runtime/cs/Quantize.cs new file mode 100644 index 00000000..22c347fe --- /dev/null +++ b/protoc-gen-bitwise/runtime/cs/Quantize.cs @@ -0,0 +1,35 @@ +// Decentraland.Networking.Bitwise — Quantize +// Copy this file into your project alongside generated *.Bitwise.cs files. +// ReSharper disable once RedundantUsingDirective + +using System; + +namespace Decentraland.Networking.Bitwise + // ReSharper disable once ArrangeNamespaceBody +{ + /// + /// Static helpers for quantizing float values to/from unsigned integers. + /// Intended for use with protobuf uint32 fields: the integer is transmitted + /// as a protobuf varint, while the float accessor lives in a generated partial class. + /// + public static class Quantize + { + /// + /// Encodes to a quantized . + /// Values outside [, ] are clamped. + /// + public static uint Encode(float value, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + var t = Math.Clamp((value - min) / (max - min), 0f, 1f); + return (uint)MathF.Round(t * steps); + } + + /// Decodes a quantized back to a float. + public static float Decode(uint encoded, float min, float max, int bits) + { + var steps = (1u << bits) - 1; + return (float)encoded / steps * (max - min) + min; + } + } +}