The Byte Order War:
Why Endianness Is the
Foundation of Every RPC System

import struct

# Machine A: x86 little-endian server
value = 1000

# Writing raw bytes WITHOUT specifying byte order
# Python defaults to native (little-endian on x86)
raw = struct.pack('i', value)    # produces: E8 03 00 00
socket.send(raw)

# ─────────────────────────────────────────

# Machine B: big-endian system
# Receives the SAME bytes: E8 03 00 00
raw = socket.recv(4)

# Interprets them as big-endian (its native format)
received = struct.unpack('i', raw)[0]
print(received)  # -402456576  ← silent corruption, no error

# '>' = big-endian (network byte order)
# '!' = equivalent alias meaning "network"

# Machine A (little-endian x86) — SERIALIZES:
value = 1000
raw = struct.pack('>i', value)   # forces big-endian: 00 00 03 E8
socket.send(raw)

# Machine B (any architecture) — DESERIALIZES:
raw = socket.recv(4)           # receives: 00 00 03 E8
value = struct.unpack('>i', raw)[0]
print(value)  # 1000 — correct on ANY machine

# Generated by thrift --gen py sensor.thrift
# You never write this — but this is what runs

def write(self, oprot):
    oprot.writeI32(self.sensor_id)    # → struct.pack('!i', ...)
    oprot.writeDouble(self.value)     # → struct.pack('!d', ...)
    oprot.writeI64(self.timestamp_ms) # → struct.pack('!q', ...)

def read(self, iprot):
    self.sensor_id    = iprot.readI32()    # → struct.unpack('!i', ...)
    self.value        = iprot.readDouble() # → struct.unpack('!d', ...)
    self.timestamp_ms = iprot.readI64()   # → struct.unpack('!q', ...)

class TBinaryProtocol:

    def writeI32(self, i32):
        # '!' = network byte order = big-endian, unconditional
        # Runs identically on x86, ARM, SPARC — no if-statements
        self.trans.write(struct.pack('!i', i32))

    def readI32(self):
        buff = self.trans.readAll(4)
        val, = struct.unpack('!i', buff)  # always big-endian
        return val

Step 1  Your code
        request = SensorReading(sensor_id=1000, value=98.6)
        client.processReading(request)

Step 2  Generated code calls write()
        request.write(oprot)

Step 3  TBinaryProtocol.writeI32(1000)
        struct.pack('!i', 1000)
        Native x86 memory : E8 03 00 00
        After '!' pack    : 00 00 03 E8  ← these 4 bytes hit the wire

Step 4  TCP transmits bytes
        Wire contains     : 00 00 03 E8

Step 5  Server (any arch) calls readI32()
        struct.unpack('!i', b'\x00\x00\x03\xe8')
        Returns           : 1000  ✓

# Varint: values stored 7 bits at a time
# High bit of each byte = continuation flag
# Byte order defined by the encoding itself — no endian decision needed

# Value 1000 = 0b1111101000
# Split into 7-bit groups from LSB: 1101000  0000111
# Add continuation bits:           11101000  00000111
# Wire bytes:                      0xE8      0x07  → just 2 bytes!

# Zigzag: handles signed ints
def zigzag_encode(n): return (n << 1) ^ (n >> 31)
# 0→0, -1→1, 1→2, -2→3  (small negatives stay small)

import ctypes

# _pack_ = 1 disables all compiler padding
# Every field lands at the exact offset you expect
# CRITICAL: both sides must use the same _pack_ value

class SensorReading(ctypes.Structure):
    _pack_ = 1
    _fields_ = [
        ("sensor_id",  ctypes.c_uint32),   # 4 bytes at offset 0
        ("timestamp",  ctypes.c_double),    # 8 bytes at offset 4
        ("value",      ctypes.c_double),    # 8 bytes at offset 12
        ("is_valid",   ctypes.c_bool),      # 1 byte  at offset 20
    ]
# Total: 21 bytes, no padding, deterministic layout

class RpcHeader(ctypes.Structure):
    _pack_ = 1
    _fields_ = [
        ("rpc_id",     ctypes.c_uint16),   # which function
        ("payload_len", ctypes.c_uint32),   # bytes that follow
    ]
# Total: 6 bytes, fixed, always read first

RPC_GET_READING = 1
RPC_SUBMIT      = 2

import socket, ctypes, threading
from shared_structs import RpcHeader, SensorReading, RPC_GET_READING

def recv_exact(sock, n):
    buf = b""
    while len(buf) < n:
        chunk = sock.recv(n - len(buf))
        if not chunk: raise RuntimeError("disconnected")
        buf += chunk
    return buf

def handle(conn):
    while True:
        # 1. Read fixed 6-byte header — no parsing, just cast
        raw_hdr = recv_exact(conn, ctypes.sizeof(RpcHeader))
        hdr = RpcHeader.from_buffer_copy(raw_hdr)   # zero-copy cast

        # 2. Read payload
        raw_payload = recv_exact(conn, hdr.payload_len)

        if hdr.rpc_id == RPC_GET_READING:
            # 3. Cast payload directly to struct — no deserialization
            req = SensorReading.from_buffer_copy(raw_payload)
            print(f"sensor={req.sensor_id} value={req.value:.2f}")

            # 4. Build response and send raw bytes — no serialization
            resp = SensorReading(sensor_id=req.sensor_id,
                                  value=req.value * 2,
                                  is_valid=True)
            conn.sendall(bytes(resp))   # raw struct bytes, no encoding

server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("", 9000))
server.listen(16)
while True:
    conn, _ = server.accept()
    threading.Thread(target=handle, args=(conn,), daemon=True).start()

import socket, ctypes
from shared_structs import RpcHeader, SensorReading, RPC_GET_READING

sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect(("10.0.0.2", 9000))   # ← across the network

# Build payload
reading = SensorReading(sensor_id=42, value=98.6, is_valid=True)
payload = bytes(reading)           # raw struct bytes, no encoding

# Send header + payload
hdr = RpcHeader(rpc_id=RPC_GET_READING,
                payload_len=len(payload))
sock.sendall(bytes(hdr) + payload)  # total: 6 + 21 = 27 bytes

# Receive response — cast directly, no parsing
raw = sock.recv(ctypes.sizeof(SensorReading))
resp = SensorReading.from_buffer_copy(raw)  # zero-copy
print(f"response value: {resp.value}")  # 197.2

# Sending SensorReading (21 bytes payload) over loopback TCP
# 100,000 iterations

Raw ctypes (zero-copy)    :  ~1.1 µs per round-trip
Thrift TCompactProtocol   :  ~4.8 µs per round-trip
Thrift TBinaryProtocol    :  ~6.2 µs per round-trip
JSON over HTTP            :  ~38  µs per round-trip

# Raw is ~4x faster than Thrift Compact for this payload size
# The gap widens with payload size
# The gap narrows when network latency dominates (cross-DC calls)

// Version 1
struct SensorReading {
  1: required i32    sensor_id,
  2: required double value,
}

// Version 2 — safely add fields
struct SensorReading {
  1: required i32    sensor_id,
  2: required double value,
  3: optional string unit,        // NEW — always optional
  // 4: DELETED 2024-01 — was legacy_calibration
  // Field ID 4 is permanently retired. Never reuse it.
  5: optional i64    recorded_at,  // NEW — gap in IDs is fine
}

from thrift.protocol import TCompactProtocol
protocol = TCompactProtocol.TCompactProtocol(transport)