Skip to content

API Reference

Remote Frame Buffer.

A transport-neutral remote framebuffer: Python produces frames, encodes them (image or H.264), streams them over WebSocket, and a browser decodes them to a canvas while sending pointer/key/resize events back.

The public API is push-based: start a server with :func:serve, then publish() frames to the returned :class:Display from your own loop and drain input with :meth:Display.poll_events.

The PyAV-dependent H.264 symbols (H264CpuEncoder, NvencCpuEncoder, h264_available, nvenc_cpu_available) are loaded lazily via :pep:562 so base (image-only) installs without the optional av dependency can still import pdum.rfb.

AdaptiveQualityController dataclass

Map observed metrics to a target quality with hysteresis + cooldown.

Source code in src/pdum/rfb/adaptive.py 
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
@dataclass
class AdaptiveQualityController:
    """Map observed metrics to a target quality with hysteresis + cooldown."""

    min_bitrate: int = 1_000_000
    max_bitrate: int = 12_000_000
    bitrate: int = 12_000_000

    min_inflight: int = 1
    max_inflight: int = 3
    inflight: int = 3

    min_fps: int = 10
    max_fps: int = 30
    fps: int = 30
    fps_step: int = 5

    # Resolution scale (deepest lever): 1.0 = full res, floored at min_scale. Absolute,
    # so the render loop applies it to its native size without compounding.
    min_scale: float = 0.5
    scale: float = 1.0
    scale_step: float = 0.25

    queue_high: int = 3  # decode_queue_size above this is "congested"
    rtt_high_ms: float = 150.0
    rtt_low_ms: float = 60.0

    cooldown_s: float = 1.0
    down_factor: float = 0.6
    up_factor: float = 1.25

    _last_change: float = -1e9

    def update(self, metrics: dict, *, now: float) -> QualityTarget | None:
        """Return a new target when a change is warranted, else ``None``."""
        if now - self._last_change < self.cooldown_s:
            return None

        queue = int(metrics.get("decode_queue_size", 0))
        rtt = float(metrics.get("rtt_ms", 0.0))
        congested = queue > self.queue_high or (rtt > 0 and rtt > self.rtt_high_ms)
        healthy = queue <= 1 and (rtt == 0 or rtt < self.rtt_low_ms)

        new_bitrate, new_inflight, new_fps = self.bitrate, self.inflight, self.fps
        new_scale = self.scale
        if congested:
            new_bitrate = max(self.min_bitrate, int(self.bitrate * self.down_factor))
            if new_bitrate == self.bitrate:  # bitrate floored; ease the frame rate
                new_fps = max(self.min_fps, self.fps - self.fps_step)
                if new_fps == self.fps:  # fps floored too; tighten latency
                    new_inflight = max(self.min_inflight, self.inflight - 1)
                    if new_inflight == self.inflight:  # everything floored; shrink resolution
                        new_scale = round(max(self.min_scale, self.scale - self.scale_step), 4)
        elif healthy:
            new_bitrate = min(self.max_bitrate, int(self.bitrate * self.up_factor))
            new_inflight = min(self.max_inflight, self.inflight + 1)
            new_fps = min(self.max_fps, self.fps + self.fps_step)
            new_scale = round(min(1.0, self.scale + self.scale_step), 4)
        else:
            return None

        if (new_bitrate, new_inflight, new_fps, new_scale) == (self.bitrate, self.inflight, self.fps, self.scale):
            return None

        self.bitrate, self.inflight, self.fps, self.scale = new_bitrate, new_inflight, new_fps, new_scale
        self._last_change = now
        return QualityTarget(bitrate=new_bitrate, max_inflight=new_inflight, fps=new_fps, scale=new_scale)

update(metrics, *, now)

Return a new target when a change is warranted, else None.

Source code in src/pdum/rfb/adaptive.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
def update(self, metrics: dict, *, now: float) -> QualityTarget | None:
    """Return a new target when a change is warranted, else ``None``."""
    if now - self._last_change < self.cooldown_s:
        return None

    queue = int(metrics.get("decode_queue_size", 0))
    rtt = float(metrics.get("rtt_ms", 0.0))
    congested = queue > self.queue_high or (rtt > 0 and rtt > self.rtt_high_ms)
    healthy = queue <= 1 and (rtt == 0 or rtt < self.rtt_low_ms)

    new_bitrate, new_inflight, new_fps = self.bitrate, self.inflight, self.fps
    new_scale = self.scale
    if congested:
        new_bitrate = max(self.min_bitrate, int(self.bitrate * self.down_factor))
        if new_bitrate == self.bitrate:  # bitrate floored; ease the frame rate
            new_fps = max(self.min_fps, self.fps - self.fps_step)
            if new_fps == self.fps:  # fps floored too; tighten latency
                new_inflight = max(self.min_inflight, self.inflight - 1)
                if new_inflight == self.inflight:  # everything floored; shrink resolution
                    new_scale = round(max(self.min_scale, self.scale - self.scale_step), 4)
    elif healthy:
        new_bitrate = min(self.max_bitrate, int(self.bitrate * self.up_factor))
        new_inflight = min(self.max_inflight, self.inflight + 1)
        new_fps = min(self.max_fps, self.fps + self.fps_step)
        new_scale = round(min(1.0, self.scale + self.scale_step), 4)
    else:
        return None

    if (new_bitrate, new_inflight, new_fps, new_scale) == (self.bitrate, self.inflight, self.fps, self.scale):
        return None

    self.bitrate, self.inflight, self.fps, self.scale = new_bitrate, new_inflight, new_fps, new_scale
    self._last_change = now
    return QualityTarget(bitrate=new_bitrate, max_inflight=new_inflight, fps=new_fps, scale=new_scale)

AuthContext dataclass

Everything the auth hook may inspect about a connecting client.

Parameters:

Name Type Description Default
token str | None

The credential from the client's hello message (v1 transport).

 None
headers Mapping[str, str] | None

Handshake request headers (e.g. Cookie), when the transport exposes them. None for the plain hello-token path.

 None
cookies Mapping[str, str] | None

Parsed request cookies, when the transport exposes them (e.g. the ASGI adapter) — the natural home for a same-origin session/OAuth cookie.

 None
path str | None

Request path including query string, when available.

 None
query Mapping[str, str] | None

Parsed query parameters, when available.

 None
remote tuple[str, int] | None

(host, port) of the peer, when available.

 None
hello dict | None

The full decoded hello dict, for transports that carry auth in-band.

 None
stream str | None

Name of the stream (named :class:~pdum.rfb.display.Display) this client is connecting to, for per-stream authorization. "default" for the single-stream serve() path; the URL-path segment for a hub (ws://host/<stream>). See :func:pdum.rfb.serve_server.

 None
Source code in src/pdum/rfb/auth.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
@dataclass(slots=True)
class AuthContext:
    """Everything the auth hook may inspect about a connecting client.

    Parameters
    ----------
    token:
        The credential from the client's ``hello`` message (v1 transport).
    headers:
        Handshake request headers (e.g. ``Cookie``), when the transport exposes
        them. ``None`` for the plain ``hello``-token path.
    cookies:
        Parsed request cookies, when the transport exposes them (e.g. the ASGI
        adapter) — the natural home for a same-origin session/OAuth cookie.
    path:
        Request path including query string, when available.
    query:
        Parsed query parameters, when available.
    remote:
        ``(host, port)`` of the peer, when available.
    hello:
        The full decoded ``hello`` dict, for transports that carry auth in-band.
    stream:
        Name of the stream (named :class:`~pdum.rfb.display.Display`) this client is
        connecting to, for per-stream authorization. ``"default"`` for the
        single-stream ``serve()`` path; the URL-path segment for a hub
        (``ws://host/<stream>``). See :func:`pdum.rfb.serve_server`.
    """

    token: str | None = None
    headers: Mapping[str, str] | None = None
    cookies: Mapping[str, str] | None = None
    path: str | None = None
    query: Mapping[str, str] | None = None
    remote: tuple[str, int] | None = None
    hello: dict | None = None
    stream: str | None = None

BackendSelection dataclass

The encoder/transport the server chose for a connection.

Source code in src/pdum/rfb/protocol.py 
180
181
182
183
184
185
186
187
@dataclass(slots=True)
class BackendSelection:
    """The encoder/transport the server chose for a connection."""

    transport: Literal["image", "h264"]
    mime: str | None = None  # for the image transport
    codec: str | None = None  # for the h264 transport, e.g. "avc1.42E01F"
    image_mode: ImageMode | None = None

Channel

Bases: Protocol

The minimal duplex byte/text channel the session drives.

Source code in src/pdum/rfb/transport.py 
18
19
20
21
22
23
24
25
26
27
28
@runtime_checkable
class Channel(Protocol):
    """The minimal duplex byte/text channel the session drives."""

    async def send(self, data: bytes | str) -> None:
        """Send one binary payload or one text control message."""
        ...

    def __aiter__(self) -> AsyncIterator[bytes | str]:
        """Asynchronously iterate inbound messages (``bytes`` or ``str``)."""
        ...

__aiter__()

Asynchronously iterate inbound messages (bytes or str).

Source code in src/pdum/rfb/transport.py 
26
27
28
def __aiter__(self) -> AsyncIterator[bytes | str]:
    """Asynchronously iterate inbound messages (``bytes`` or ``str``)."""
    ...

send(data) async

Send one binary payload or one text control message.

Source code in src/pdum/rfb/transport.py 
22
23
24
async def send(self, data: bytes | str) -> None:
    """Send one binary payload or one text control message."""
    ...

ColorSpace dataclass

A small, explicit display-referred color descriptor for a frame/stream.

Mirrors the WebCodecs VideoColorSpace fields the browser consumes directly (primaries gamut, transfer function, matrix RGB-vs-YUV coupling) plus a full_range flag and a bit_depth for the HDR future. Two presets ship as first-class — :data:SRGB (the implicit default) and :data:DISPLAY_P3 (Apple wide-gamut SDR, 8-bit); bt2020/pq/hlg are expressible (HDR is designed-for) but not yet wired through a 10-bit pipeline.

The library tags color; it does not convert. The upstream renderer is responsible for producing pixels already in the declared space.

Source code in src/pdum/rfb/types.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
@dataclass(slots=True, frozen=True)
class ColorSpace:
    """A small, explicit display-referred color descriptor for a frame/stream.

    Mirrors the WebCodecs ``VideoColorSpace`` fields the browser consumes directly
    (``primaries`` gamut, ``transfer`` function, ``matrix`` RGB-vs-YUV coupling) plus a
    ``full_range`` flag and a ``bit_depth`` for the HDR future. Two presets ship as
    first-class — :data:`SRGB` (the implicit default) and :data:`DISPLAY_P3` (Apple
    wide-gamut SDR, 8-bit); ``bt2020``/``pq``/``hlg`` are expressible (HDR is designed-for)
    but not yet wired through a 10-bit pipeline.

    The library **tags** color; it does not convert. The upstream renderer is responsible
    for producing pixels already in the declared space.
    """

    primaries: Primaries = "bt709"
    transfer: Transfer = "srgb"
    matrix: ColorMatrix = "rgb"
    full_range: bool = True
    bit_depth: int = 8

    def to_dict(self) -> dict:
        """The wire/JSON form (as carried on frame headers and the ``config`` message)."""
        return asdict(self)

to_dict()

The wire/JSON form (as carried on frame headers and the config message).

Source code in src/pdum/rfb/types.py 
50
51
52
def to_dict(self) -> dict:
    """The wire/JSON form (as carried on frame headers and the ``config`` message)."""
    return asdict(self)

Display

A single shared framebuffer that one or more browsers attach to.

Parameters:

Name Type Description Default
width int

Initial framebuffer size. Updated automatically whenever you publish a differently-shaped frame.

 required
height int

Initial framebuffer size. Updated automatically whenever you publish a differently-shaped frame.

 required
fps int

Advisory frame rate (used as the encoder's IDR cadence / metrics target); the actual cadence is whatever your publish loop does.

 30
record_events bool

Also accumulate raw events in :attr:recorded (exposed via the server's GET /recorded-events side channel and the headless e2e harness).

 False
event_log str | Path | None

Optional path; received events are appended as JSON lines.

 None
event_queue_size int

Bound on the un-polled event backlog; the oldest events are dropped when a publisher never calls :meth:poll_events.

 4096
own_frames bool

Opt in to server-owned frames. By default :meth:publish borrows the caller's buffer (zero-copy) and reads it asynchronously, so you must publish a fresh buffer each call (or not mutate it until it is encoded). With own_frames=True each published frame is copied into a server-owned, recycled buffer on the publish thread, so you may reuse/mutate your own buffer immediately after :meth:publish returns — no reallocation and no "frame released" callback. Supported for cpu and cuda frames; metal raises (MLX arrays are immutable, so the borrow contract already holds). See :meth:publish.

 False
resize_policy str

"publisher" (default) — you own the render size and a viewer's set_viewport is informational. "match_client" — the render stream follows the viewer: the latest set_viewport becomes :attr:target_size (last-writer-wins across viewers), which your render loop reads to size the next frame.

 'publisher'
max_render_dimension int | None

Cap on either dimension of a match_client :attr:target_size (AR-preserving), guarding against a maximized 4K window forcing a huge encode. None = no cap.

 None
resize_debounce float

Seconds a match_client target must be stable before it surfaces through :attr:target_size, so a drag-resize doesn't storm the encoder rebuild (default 0.12).

 0.12
clock Callable[[], float] | None

Monotonic clock returning seconds; injectable for deterministic tests.

 None
Source code in src/pdum/rfb/display.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
class Display:
    """A single shared framebuffer that one or more browsers attach to.

    Parameters
    ----------
    width, height:
        Initial framebuffer size. Updated automatically whenever you publish a
        differently-shaped frame.
    fps:
        Advisory frame rate (used as the encoder's IDR cadence / metrics target);
        the *actual* cadence is whatever your publish loop does.
    record_events:
        Also accumulate raw events in :attr:`recorded` (exposed via the server's
        ``GET /recorded-events`` side channel and the headless e2e harness).
    event_log:
        Optional path; received events are appended as JSON lines.
    event_queue_size:
        Bound on the un-polled event backlog; the **oldest** events are dropped
        when a publisher never calls :meth:`poll_events`.
    own_frames:
        Opt in to **server-owned frames**. By default :meth:`publish` *borrows* the
        caller's buffer (zero-copy) and reads it asynchronously, so you must publish a
        fresh buffer each call (or not mutate it until it is encoded). With
        ``own_frames=True`` each published frame is **copied into a server-owned,
        recycled buffer** on the publish thread, so you may reuse/mutate your own buffer
        immediately after :meth:`publish` returns — no reallocation and no "frame
        released" callback. Supported for ``cpu`` and ``cuda`` frames; ``metal`` raises
        (MLX arrays are immutable, so the borrow contract already holds). See
        :meth:`publish`.
    resize_policy:
        ``"publisher"`` (default) — you own the render size and a viewer's ``set_viewport``
        is informational. ``"match_client"`` — the render stream *follows the viewer*: the
        latest ``set_viewport`` becomes :attr:`target_size` (last-writer-wins across viewers),
        which your render loop reads to size the next frame.
    max_render_dimension:
        Cap on either dimension of a ``match_client`` :attr:`target_size` (AR-preserving),
        guarding against a maximized 4K window forcing a huge encode. ``None`` = no cap.
    resize_debounce:
        Seconds a ``match_client`` target must be stable before it surfaces through
        :attr:`target_size`, so a drag-resize doesn't storm the encoder rebuild (default 0.12).
    clock:
        Monotonic clock returning seconds; injectable for deterministic tests.
    """

    def __init__(
        self,
        width: int,
        height: int,
        *,
        fps: int = 30,
        record_events: bool = False,
        event_log: str | Path | None = None,
        event_queue_size: int = 4096,
        own_frames: bool = False,
        resize_policy: str = "publisher",
        max_render_dimension: int | None = None,
        resize_debounce: float = 0.12,
        clock: Callable[[], float] | None = None,
    ) -> None:
        self.width = int(width)
        self.height = int(height)
        # The initial (native/full-resolution) size, kept fixed even as publish() resizes
        # width/height. It is the base a DownscaleHint scales off, so an absolute scale
        # never compounds when the render loop honors the hint by publishing smaller.
        self._base_width = int(width)
        self._base_height = int(height)
        self.fps = fps
        self._clock = clock or time.monotonic
        self._start = self._clock()

        # "match-client" resize policy: when enabled, a viewer's set_viewport becomes a
        # *target size* the render loop follows (default "publisher" = you own the size,
        # set_viewport is informational). Last-writer-wins across viewers; debounced so a
        # drag-resize doesn't storm the encoder rebuild; clamped to max_render_dimension.
        if resize_policy not in ("publisher", "match_client"):
            raise ValueError(f"resize_policy must be 'publisher' or 'match_client', got {resize_policy!r}")
        self.resize_policy = resize_policy
        self.max_render_dimension = max_render_dimension
        self._resize_debounce = float(resize_debounce)
        self._pending_target: tuple[int, int] | None = None
        self._pending_ratio = 1.0
        self._pending_at = 0.0
        self._committed_target: tuple[int, int] | None = None
        self._committed_ratio = 1.0

        self._latest: RawFrame | None = None
        self._version = 0
        # Opt-in frame ownership (own_frames=True): publish() copies each frame into a
        # server-owned buffer drawn from this recycled pool, so the caller may reuse its
        # buffer immediately. Empty/unused when own_frames is False. See _own_copy / _take_owned.
        self._own_frames = bool(own_frames)
        self._own_pool: list[Any] = []
        self._own_key: tuple[Any, ...] | None = None
        self._feeds: set[_ClientFeed] = set()
        self._sessions: set[RfbSession] = set()
        self._clients: dict[str, _ClientFeed] = {}
        # Frame taps that are *not* viewers: server-side recordings (see record()). They are
        # woken by publish() like feeds but excluded from client_count and the adaptive
        # downscale aggregation.
        self._taps: set[Any] = set()

        # Adaptive resolution (serve(adaptive=True)): the effective render scale is the
        # minimum any connected viewer's controller currently wants (the most-congested
        # viewer wins); a change enqueues a DownscaleHint through poll_events().
        self._effective_scale = 1.0

        self._events: deque[InputEvent | DownscaleHint] = deque(maxlen=event_queue_size)
        self._events_signal = asyncio.Event()

        self._record_events = record_events or event_log is not None
        self._event_log = Path(event_log) if event_log else None
        self.recorded: list[EventDict] = []

        self._closed = False
        # Set by serve() so aclose() can stop the listener it started.
        self._server: Any = None
        self._server_cm: Any = None
        # Set by Server.add_stream() so display.server.add_stream(...) works.
        self._owner_server: Any = None
        # The URL path segment this stream is reached at (set by Server.add_stream()).
        self._stream_name: str = "default"

    # --- publishing --------------------------------------------------------

    def publish(
        self,
        frame: np.ndarray | RawFrame | Any,
        *,
        pixel_ratio: float | None = None,
        color: Any = None,
    ) -> None:
        """Make ``frame`` the latest frame and wake every connected viewer.

        Synchronous and non-blocking. ``frame`` may be:

        * a contiguous host ``uint8`` array — ``(H, W, 3)`` ``rgb24`` or
          ``(H, W, 4)`` ``rgba8``;
        * a **CUDA tensor** exposing ``__cuda_array_interface__`` (e.g. CuPy) of
          shape ``(H, W, 3|4)`` — published as a zero-copy ``cuda`` frame (for
          NV12, or other frameworks, build a ``RawFrame`` via
          :func:`pdum.rfb.gpu.cuda_frame`);
        * an **MLX (Apple Metal) array** of shape ``(H, W, 3|4)`` — published as a
          ``metal`` frame; the VideoToolbox encoder converts RGB(A)→NV12 on the GPU
          (for pre-converted NV12, use :func:`pdum.rfb.metal.metal_frame`);
        * a ready :class:`~pdum.rfb.types.RawFrame` (any ``memory``).

        Latest-frame-wins: a viewer that is behind simply skips intermediate frames.

        **Ownership.** By default ``publish()`` *borrows* your buffer — it stores a bare
        reference and reads the pixels **asynchronously**, on each viewer's encode worker
        thread. The borrow window runs from here until every viewer has finished encoding
        the frame, and it is **widest under** ``still_after`` (the resting frame is re-read
        ~``still_after`` seconds later for the lossless still). So in borrow mode, publish a
        *fresh* buffer each call, or do not mutate a published buffer until it is encoded.
        Construct the ``Display`` (or call :func:`~pdum.rfb.serve`) with ``own_frames=True``
        to instead have the server copy each frame into a recycled buffer, after which you
        may reuse/mutate your own buffer immediately (``cpu``/``cuda`` only; ``metal`` raises).

        Parameters
        ----------
        pixel_ratio:
            Render-side DPR for this frame (device px per logical px). ``None`` keeps a
            :class:`~pdum.rfb.types.RawFrame`'s own value (else ``1.0``). See
            :attr:`~pdum.rfb.types.RawFrame.pixel_ratio`.
        color:
            Color descriptor for this frame — a :class:`~pdum.rfb.types.ColorSpace`, its
            ``dict`` form, or ``None`` (sRGB). The renderer must already produce pixels in
            the declared space; the library only tags them.
        """
        if self._closed:
            raise RuntimeError("publish() called on a closed Display")

        frame_pr = frame.pixel_ratio if isinstance(frame, RawFrame) else 1.0
        frame_color = frame.color if isinstance(frame, RawFrame) else None
        resolved_pr = float(frame_pr if pixel_ratio is None else pixel_ratio)
        resolved_color = frame_color if color is None else _color_to_dict(color)

        if isinstance(frame, RawFrame):
            data, width, height = frame.data, frame.width, frame.height
            pixel_format, memory = frame.pixel_format, frame.memory
        elif isinstance(frame, np.ndarray):
            if frame.ndim != 3 or frame.shape[2] not in (3, 4):
                raise ValueError(f"unsupported frame shape {frame.shape!r}; expected (H, W, 3) or (H, W, 4)")
            height, width = int(frame.shape[0]), int(frame.shape[1])
            pixel_format = "rgb24" if frame.shape[2] == 3 else "rgba8"
            data, memory = frame, "cpu"
        elif _is_cuda_tensor(frame):
            shape = getattr(frame, "shape", None)
            if shape is None or len(shape) != 3 or shape[2] not in (3, 4):
                raise ValueError("CUDA publish expects an (H, W, 3|4) tensor; use pdum.rfb.gpu.cuda_frame() for NV12")
            height, width = int(shape[0]), int(shape[1])
            pixel_format = "rgb24" if shape[2] == 3 else "rgba8"
            data, memory = frame, "cuda"
        elif _is_metal_tensor(frame):
            shape = getattr(frame, "shape", None)
            if shape is None or len(shape) != 3 or shape[2] not in (3, 4):
                raise ValueError(
                    "Metal publish expects an (H, W, 3|4) MLX array; use pdum.rfb.metal.metal_frame() for NV12"
                )
            height, width = int(shape[0]), int(shape[1])
            pixel_format = "rgb24" if shape[2] == 3 else "rgba8"
            data, memory = frame, "metal"
        else:
            raise TypeError("publish() expects a numpy.ndarray, a CUDA/Metal tensor, or a RawFrame")

        if memory == "metal":
            # Materialize the lazy MLX render on *this* (loop) thread: MLX binds a lazy graph to
            # its origin thread's stream, so the session's encode worker thread cannot evaluate a
            # frame built here. The GPU NV12 conversion still runs on the worker. See metal.materialize.
            from .metal import materialize

            materialize(data)

        if self._own_frames:
            # Server-owned mode: copy into a recycled buffer on this (loop) thread so the caller
            # may reuse/mutate its own buffer immediately. Severs the async-read aliasing entirely.
            data = self._own_copy(data, memory)

        timestamp_us = int((self._clock() - self._start) * 1_000_000)
        # seq is a placeholder; each feed stamps its own per-client sequence.
        self._latest = RawFrame(
            seq=0,
            width=width,
            height=height,
            timestamp_us=timestamp_us,
            pixel_format=pixel_format,  # type: ignore[arg-type]
            memory=memory,  # type: ignore[arg-type]
            data=data,
            pixel_ratio=resolved_pr,
            color=resolved_color,
        )
        self.width, self.height = width, height
        self._version += 1
        for feed in self._feeds:
            feed._wake()
        for tap in self._taps:
            tap._wake()

    def _own_copy(self, data: Any, memory: str) -> Any:
        """Copy ``data`` into a server-owned buffer (``own_frames`` mode) so the caller may
        reuse its own buffer immediately. Runs on the loop thread. ``cpu`` → numpy, ``cuda`` →
        CuPy device-to-device; ``metal`` is unsupported (MLX is immutable — borrow already holds)."""
        if memory == "cpu":
            arr = np.asarray(data)
            buf = self._take_owned(arr.shape, arr.dtype, memory)
            np.copyto(buf, arr)
            return buf
        if memory == "cuda":
            import cupy as cp  # lazy: only when a cuda frame is published under own_frames

            src = cp.asarray(data)
            buf = self._take_owned(src.shape, src.dtype, memory)
            cp.copyto(buf, src)
            return buf
        raise NotImplementedError(
            "own_frames is not supported for Metal frames; publish a fresh mx.array per frame "
            "(MLX arrays are immutable, so the borrow contract already holds). See docs/metal_videotoolbox.md."
        )

    def _take_owned(self, shape: tuple[int, ...], dtype: Any, memory: str) -> Any:
        """Return a reusable server-owned buffer of ``(shape, dtype, memory)`` that no in-flight
        frame still references. A size/dtype/memory change drops the pool (reallocate, like the
        encoder rebuild on resize).

        Correctness rests on CPython refcounting: while a session holds ``frame =
        replace(latest, seq)`` across its off-thread encode, that buffer's refcount is elevated,
        so it is skipped here and never overwritten mid-encode. Steady state the pool stabilizes
        at ~(concurrently-encoding viewers + 1) buffers and recycles them — no per-frame alloc."""
        key = (shape, dtype, memory)
        if key != self._own_key:
            self._own_pool = []
            self._own_key = key
        pool = self._own_pool
        for i in range(len(pool)):
            # getrefcount == 2 means the only refs are the pool slot + getrefcount's own argument,
            # i.e. nothing else holds it. Index as pool[i] (no bound local), or a temporary local
            # would add one and mask a free buffer.
            if sys.getrefcount(pool[i]) <= 2:
                return pool[i]
        if memory == "cuda":
            import cupy as cp

            buf = cp.empty(shape, dtype)
        else:
            buf = np.empty(shape, dtype)  # C-contiguous regardless of caller layout
        pool.append(buf)
        return buf

    # --- recording ---------------------------------------------------------

    def record(
        self,
        path: str | Path,
        *,
        fps: int | None = None,
        bitrate: int | None = None,
        crf: int | None = None,
        preset: str = "veryfast",
    ) -> Recording:
        """Record the published-frame stream to a real **MP4** file (server-side tap).

        Returns a started :class:`~pdum.rfb.recording.Recording` — an async context
        manager / handle with :meth:`~pdum.rfb.recording.Recording.stop`. It taps the
        same latest-frame stream the viewer sessions pull from, so it is **independent
        of any connected browser** and works fully headless (demos, CI artifacts,
        golden-frame capture). Frames are H.264-encoded (libx264) and muxed to
        **AVCC-in-MP4** via a **separate** PyAV path — this is the one place AVCC/MP4 is
        correct; it never touches the Annex-B WebSocket payload.

        The real, monotonic per-frame timestamps are honored, so a variable-cadence
        publisher (sparse ``still_after`` scenes included) records with correct frame
        timing. Call it on the event-loop thread; it starts a background encode task.

        ::

            async with display.record("out.mp4"):
                for _ in range(300):
                    display.publish(render()); await asyncio.sleep(1 / 30)
            # file finalized on exit — or use rec = display.record(...); await rec.stop()

        Parameters
        ----------
        path:
            Output ``.mp4`` file path.
        fps:
            Advisory frame rate for the encoder's rate control (default: the display's
            ``fps``). The *actual* timing comes from the frames' real timestamps.
        bitrate:
            Target bitrate in bits/s (average). Mutually exclusive with ``crf``; if
            neither is given, a sensible constant-quality ``crf`` is used.
        crf:
            Constant Rate Factor (0–51, lower = better) for constant-quality encoding.
        preset:
            libx264 speed/quality preset (default ``"veryfast"``).

        Raises
        ------
        RuntimeError
            If PyAV (the ``[h264]`` extra) is not installed.
        """
        from .recording import Recording

        rec = Recording(self, path, fps=fps or self.fps, bitrate=bitrate, crf=crf, preset=preset)
        rec._start_task()
        return rec

    # --- events ------------------------------------------------------------

    def poll_events(self) -> list[InputEvent | DownscaleHint]:
        """Drain and return everything queued since the last poll.

        The list is ordinarily :class:`~pdum.rfb.types.InputEvent`\\ s (input from all
        connected viewers). With ``serve(adaptive=True)`` it may also carry a
        :class:`~pdum.rfb.types.DownscaleHint` — a server-driven resolution suggestion —
        interleaved in arrival order; tell them apart with ``isinstance``.
        """
        out = list(self._events)
        self._events.clear()
        return out

    async def events(self) -> AsyncIterator[InputEvent | DownscaleHint]:
        """Async-iterate queued events (alternative to :meth:`poll_events`).

        Yields :class:`~pdum.rfb.types.InputEvent`\\ s and, under
        ``serve(adaptive=True)``, :class:`~pdum.rfb.types.DownscaleHint`\\ s. Use this or
        :meth:`poll_events` — both drain the same queue.
        """
        while not self._closed:
            if self._events:
                yield self._events.popleft()
                continue
            self._events_signal.clear()
            await self._events_signal.wait()

    @property
    def client_count(self) -> int:
        """Number of currently connected viewers."""
        return len(self._feeds)

    @property
    def pixel_ratio(self) -> float:
        """Render-side DPR of the latest published frame (``1.0`` before the first publish)."""
        return self._latest.pixel_ratio if self._latest is not None else 1.0

    @property
    def color(self) -> dict | None:
        """Color descriptor of the latest published frame, or ``None`` (sRGB)."""
        return self._latest.color if self._latest is not None else None

    # --- match-client resize (opt-in) --------------------------------------

    def _clamp_render_size(self, w: int, h: int) -> tuple[int, int]:
        """Clamp a requested render size to ``max_render_dimension`` (AR-preserving) and to
        even, >= 2 dimensions (H.264 / NV12 need even). ``max_render_dimension=None`` = no cap."""
        w, h = max(2, int(w)), max(2, int(h))
        cap = self.max_render_dimension
        if cap is not None and max(w, h) > cap:
            scale = cap / max(w, h)
            w, h = max(2, round(w * scale)), max(2, round(h * scale))
        return (w - (w % 2), h - (h % 2))

    def _request_target(self, pw: int, ph: int, ratio: float) -> None:
        """Record a viewer's requested render size (``match_client`` only). Debounced: the
        value surfaces through :attr:`target_size` after it has been stable for
        ``resize_debounce`` seconds, so a drag-resize doesn't storm the encoder rebuild."""
        size = self._clamp_render_size(pw, ph)
        if size != self._pending_target:
            self._pending_target = size
            self._pending_ratio = float(ratio)
            self._pending_at = self._clock()

    def _settle_target(self) -> None:
        if self._pending_target is not None and (self._clock() - self._pending_at) >= self._resize_debounce:
            self._committed_target = self._pending_target
            self._committed_ratio = self._pending_ratio
            self._pending_target = None

    @property
    def target_size(self) -> tuple[int, int] | None:
        """Latest client-requested render size under ``resize_policy="match_client"`` (debounced,
        clamped, even dims), or ``None`` before any viewport arrives / in ``"publisher"`` mode.

        Read it in your render loop to follow the viewer::

            w, h = display.target_size or (display.width, display.height)
            display.publish(render(state, w, h), pixel_ratio=display.target_ratio)
        """
        self._settle_target()
        return self._committed_target

    @property
    def target_ratio(self) -> float:
        """The client DPR that accompanies :attr:`target_size` (``1.0`` until one arrives)."""
        self._settle_target()
        return self._committed_ratio

    @property
    def port(self) -> int | None:
        """The bound TCP port (useful when serving with ``port=0``)."""
        if self._server is None:
            return None
        return next(iter(self._server.sockets)).getsockname()[1]

    @property
    def server(self) -> Any:
        """The owning :class:`~pdum.rfb.server.Server` hub, if this is a stream of one.

        Lets the convenience ``display = await serve(...)`` path reach the hub to add
        more streams: ``display.server.add_stream("camera_b", 640, 480)``. ``None``
        for a bare ``Display`` constructed directly.
        """
        return self._owner_server

    @property
    def ws_url(self) -> str:
        """Browser-reachable WebSocket URL for this stream (e.g. ``ws://host:port/name``).

        Raises if the server is not bound yet (call ``await rfb.serve(...)`` first). A
        wildcard bind host (``0.0.0.0``/``::``) is reported as ``127.0.0.1``.
        """
        if self.port is None:
            raise RuntimeError("Display is not serving yet; call await rfb.serve(...) first")
        host = getattr(self._owner_server, "host", None) or "127.0.0.1"
        if host in ("0.0.0.0", "", "::"):
            host = "127.0.0.1"
        return f"ws://{host}:{self.port}/{self._stream_name}"

    def widget(
        self,
        *,
        batteries: bool = True,
        base_path: str | None = None,
        host: str | None = None,
        **chrome: Any,
    ) -> Any:
        """Return an anywidget viewer bound to this stream (needs the ``[anywidget]`` extra).

        In a notebook: ``display.widget()`` renders the batteries viewer;
        ``display.widget(batteries=False)`` the bare canvas. One widget = one Web Worker +
        one WebSocket; the server multiplexes many streams on one port. For remote/HTTPS
        notebooks, mount ``pdum.rfb.asgi.rfb_hub_endpoint`` same-origin and pass
        ``base_path=`` (the widget then uses a same-origin ``wss://`` URL). Extra keyword
        args (e.g. ``show_toolbar=False``) become widget traits.

        Raises if the server is not bound yet.
        """
        from .notebook import RfbCanvas, RfbViewer

        if self.port is None:
            raise RuntimeError("Display is not serving yet; call await rfb.serve(...) first")
        resolved_host = host if host is not None else (getattr(self._owner_server, "host", None) or "127.0.0.1")
        if resolved_host in ("0.0.0.0", "", "::"):
            # Let the browser use the page's own hostname (correct for remote notebooks).
            resolved_host = "auto"
        cls = RfbViewer if batteries else RfbCanvas
        kwargs: dict[str, Any] = {"port": self.port, "stream": self._stream_name, "host": resolved_host, **chrome}
        if base_path is not None:
            kwargs["base_path"] = base_path
        return cls(**kwargs)

    # --- lifecycle ---------------------------------------------------------

    def _close_local(self) -> None:
        """Disconnect viewers and wake waiters, *without* stopping any listener.

        The per-stream half of teardown: a hub (:class:`~pdum.rfb.server.Server`)
        calls this on each stream while it stops the shared listener once.
        """
        if self._closed:
            return
        self._closed = True
        for feed in list(self._feeds):
            feed.close()
        for tap in list(self._taps):
            tap.close()
        self._events_signal.set()

    async def aclose(self) -> None:
        """Stop the server, disconnect viewers, and release encoder resources."""
        self._close_local()
        if self._server_cm is not None:
            cm, self._server_cm = self._server_cm, None
            self._server = None
            await cm.__aexit__(None, None, None)

    # --- internal (used by the connection server) --------------------------

    def _enqueue_event(self, client_id: str, principal: Principal | None, event: EventDict) -> None:
        received_us = int((self._clock() - self._start) * 1_000_000)
        self._events.append(InputEvent(client_id=client_id, principal=principal, event=event, received_us=received_us))
        self._events_signal.set()
        if self._record_events:
            self.recorded.append(event)
            if self._event_log is not None:
                with self._event_log.open("a") as fh:
                    fh.write(json.dumps(event) + "\n")

    def _add_tap(self, tap: Any) -> None:
        """Register a non-viewer frame tap (e.g. a :class:`Recording`); publish() wakes it."""
        self._taps.add(tap)

    def _remove_tap(self, tap: Any) -> None:
        self._taps.discard(tap)

    def _recompute_render_hint(self) -> None:
        """Recompute the aggregate adaptive render scale and enqueue a
        :class:`~pdum.rfb.types.DownscaleHint` when it changes.

        The effective scale is the **minimum** any connected viewer's controller wants,
        so one congested viewer downscales the shared stream for everyone; when that
        viewer recovers or disconnects the scale climbs back and a recovery hint fires.
        No-op unless the value actually changed (de-duped, so it never spams the queue).
        """
        effective = min((f.render_scale for f in self._feeds), default=1.0)
        if effective == self._effective_scale:
            return
        self._effective_scale = effective
        w = max(2, round(self._base_width * effective))
        h = max(2, round(self._base_height * effective))
        w -= w % 2
        h -= h % 2
        received_us = int((self._clock() - self._start) * 1_000_000)
        self._events.append(DownscaleHint(scale=effective, width=w, height=h, received_us=received_us))
        self._events_signal.set()

    def _make_feed(self, client_id: str, principal: Principal | None) -> _ClientFeed:
        feed = _ClientFeed(self, client_id, principal)
        self._feeds.add(feed)
        self._clients[client_id] = feed
        return feed

    def _register_session(self, session: RfbSession) -> None:
        self._sessions.add(session)

    def _remove(self, client_id: str, feed: _ClientFeed, session: RfbSession | None) -> None:
        self._feeds.discard(feed)
        self._clients.pop(client_id, None)
        if session is not None:
            self._sessions.discard(session)
        # A viewer leaving can relax the aggregate downscale (its congestion is gone).
        self._recompute_render_hint()

client_count property

Number of currently connected viewers.

color property

Color descriptor of the latest published frame, or None (sRGB).

pixel_ratio property

Render-side DPR of the latest published frame (1.0 before the first publish).

port property

The bound TCP port (useful when serving with port=0).

server property

The owning :class:~pdum.rfb.server.Server hub, if this is a stream of one.

Lets the convenience display = await serve(...) path reach the hub to add more streams: display.server.add_stream("camera_b", 640, 480). None for a bare Display constructed directly.

target_ratio property

The client DPR that accompanies :attr:target_size (1.0 until one arrives).

target_size property

Latest client-requested render size under resize_policy="match_client" (debounced, clamped, even dims), or None before any viewport arrives / in "publisher" mode.

Read it in your render loop to follow the viewer::

w, h = display.target_size or (display.width, display.height)
display.publish(render(state, w, h), pixel_ratio=display.target_ratio)

ws_url property

Browser-reachable WebSocket URL for this stream (e.g. ws://host:port/name).

Raises if the server is not bound yet (call await rfb.serve(...) first). A wildcard bind host (0.0.0.0/::) is reported as 127.0.0.1.

aclose() async

Stop the server, disconnect viewers, and release encoder resources.

Source code in src/pdum/rfb/display.py 
600
601
602
603
604
605
606
async def aclose(self) -> None:
    """Stop the server, disconnect viewers, and release encoder resources."""
    self._close_local()
    if self._server_cm is not None:
        cm, self._server_cm = self._server_cm, None
        self._server = None
        await cm.__aexit__(None, None, None)

events() async

Async-iterate queued events (alternative to :meth:poll_events).

Yields :class:~pdum.rfb.types.InputEvent\ s and, under serve(adaptive=True), :class:~pdum.rfb.types.DownscaleHint\ s. Use this or :meth:poll_events — both drain the same queue.

Source code in src/pdum/rfb/display.py 
443
444
445
446
447
448
449
450
451
452
453
454
455
async def events(self) -> AsyncIterator[InputEvent | DownscaleHint]:
    """Async-iterate queued events (alternative to :meth:`poll_events`).

    Yields :class:`~pdum.rfb.types.InputEvent`\\ s and, under
    ``serve(adaptive=True)``, :class:`~pdum.rfb.types.DownscaleHint`\\ s. Use this or
    :meth:`poll_events` — both drain the same queue.
    """
    while not self._closed:
        if self._events:
            yield self._events.popleft()
            continue
        self._events_signal.clear()
        await self._events_signal.wait()

poll_events()

Drain and return everything queued since the last poll.

The list is ordinarily :class:~pdum.rfb.types.InputEvent\ s (input from all connected viewers). With serve(adaptive=True) it may also carry a :class:~pdum.rfb.types.DownscaleHint — a server-driven resolution suggestion — interleaved in arrival order; tell them apart with isinstance.

Source code in src/pdum/rfb/display.py 
431
432
433
434
435
436
437
438
439
440
441
def poll_events(self) -> list[InputEvent | DownscaleHint]:
    """Drain and return everything queued since the last poll.

    The list is ordinarily :class:`~pdum.rfb.types.InputEvent`\\ s (input from all
    connected viewers). With ``serve(adaptive=True)`` it may also carry a
    :class:`~pdum.rfb.types.DownscaleHint` — a server-driven resolution suggestion —
    interleaved in arrival order; tell them apart with ``isinstance``.
    """
    out = list(self._events)
    self._events.clear()
    return out

publish(frame, *, pixel_ratio=None, color=None)

Make frame the latest frame and wake every connected viewer.

Synchronous and non-blocking. frame may be:

  • a contiguous host uint8 array — (H, W, 3) rgb24 or (H, W, 4) rgba8;
  • a CUDA tensor exposing __cuda_array_interface__ (e.g. CuPy) of shape (H, W, 3|4) — published as a zero-copy cuda frame (for NV12, or other frameworks, build a RawFrame via :func:pdum.rfb.gpu.cuda_frame);
  • an MLX (Apple Metal) array of shape (H, W, 3|4) — published as a metal frame; the VideoToolbox encoder converts RGB(A)→NV12 on the GPU (for pre-converted NV12, use :func:pdum.rfb.metal.metal_frame);
  • a ready :class:~pdum.rfb.types.RawFrame (any memory).

Latest-frame-wins: a viewer that is behind simply skips intermediate frames.

Ownership. By default publish() borrows your buffer — it stores a bare reference and reads the pixels asynchronously, on each viewer's encode worker thread. The borrow window runs from here until every viewer has finished encoding the frame, and it is widest under still_after (the resting frame is re-read ~still_after seconds later for the lossless still). So in borrow mode, publish a fresh buffer each call, or do not mutate a published buffer until it is encoded. Construct the Display (or call :func:~pdum.rfb.serve) with own_frames=True to instead have the server copy each frame into a recycled buffer, after which you may reuse/mutate your own buffer immediately (cpu/cuda only; metal raises).

Parameters:

Name Type Description Default
pixel_ratio float | None

Render-side DPR for this frame (device px per logical px). None keeps a :class:~pdum.rfb.types.RawFrame's own value (else 1.0). See :attr:~pdum.rfb.types.RawFrame.pixel_ratio.

 None
color Any

Color descriptor for this frame — a :class:~pdum.rfb.types.ColorSpace, its dict form, or None (sRGB). The renderer must already produce pixels in the declared space; the library only tags them.

 None
Source code in src/pdum/rfb/display.py 
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
def publish(
    self,
    frame: np.ndarray | RawFrame | Any,
    *,
    pixel_ratio: float | None = None,
    color: Any = None,
) -> None:
    """Make ``frame`` the latest frame and wake every connected viewer.

    Synchronous and non-blocking. ``frame`` may be:

    * a contiguous host ``uint8`` array — ``(H, W, 3)`` ``rgb24`` or
      ``(H, W, 4)`` ``rgba8``;
    * a **CUDA tensor** exposing ``__cuda_array_interface__`` (e.g. CuPy) of
      shape ``(H, W, 3|4)`` — published as a zero-copy ``cuda`` frame (for
      NV12, or other frameworks, build a ``RawFrame`` via
      :func:`pdum.rfb.gpu.cuda_frame`);
    * an **MLX (Apple Metal) array** of shape ``(H, W, 3|4)`` — published as a
      ``metal`` frame; the VideoToolbox encoder converts RGB(A)→NV12 on the GPU
      (for pre-converted NV12, use :func:`pdum.rfb.metal.metal_frame`);
    * a ready :class:`~pdum.rfb.types.RawFrame` (any ``memory``).

    Latest-frame-wins: a viewer that is behind simply skips intermediate frames.

    **Ownership.** By default ``publish()`` *borrows* your buffer — it stores a bare
    reference and reads the pixels **asynchronously**, on each viewer's encode worker
    thread. The borrow window runs from here until every viewer has finished encoding
    the frame, and it is **widest under** ``still_after`` (the resting frame is re-read
    ~``still_after`` seconds later for the lossless still). So in borrow mode, publish a
    *fresh* buffer each call, or do not mutate a published buffer until it is encoded.
    Construct the ``Display`` (or call :func:`~pdum.rfb.serve`) with ``own_frames=True``
    to instead have the server copy each frame into a recycled buffer, after which you
    may reuse/mutate your own buffer immediately (``cpu``/``cuda`` only; ``metal`` raises).

    Parameters
    ----------
    pixel_ratio:
        Render-side DPR for this frame (device px per logical px). ``None`` keeps a
        :class:`~pdum.rfb.types.RawFrame`'s own value (else ``1.0``). See
        :attr:`~pdum.rfb.types.RawFrame.pixel_ratio`.
    color:
        Color descriptor for this frame — a :class:`~pdum.rfb.types.ColorSpace`, its
        ``dict`` form, or ``None`` (sRGB). The renderer must already produce pixels in
        the declared space; the library only tags them.
    """
    if self._closed:
        raise RuntimeError("publish() called on a closed Display")

    frame_pr = frame.pixel_ratio if isinstance(frame, RawFrame) else 1.0
    frame_color = frame.color if isinstance(frame, RawFrame) else None
    resolved_pr = float(frame_pr if pixel_ratio is None else pixel_ratio)
    resolved_color = frame_color if color is None else _color_to_dict(color)

    if isinstance(frame, RawFrame):
        data, width, height = frame.data, frame.width, frame.height
        pixel_format, memory = frame.pixel_format, frame.memory
    elif isinstance(frame, np.ndarray):
        if frame.ndim != 3 or frame.shape[2] not in (3, 4):
            raise ValueError(f"unsupported frame shape {frame.shape!r}; expected (H, W, 3) or (H, W, 4)")
        height, width = int(frame.shape[0]), int(frame.shape[1])
        pixel_format = "rgb24" if frame.shape[2] == 3 else "rgba8"
        data, memory = frame, "cpu"
    elif _is_cuda_tensor(frame):
        shape = getattr(frame, "shape", None)
        if shape is None or len(shape) != 3 or shape[2] not in (3, 4):
            raise ValueError("CUDA publish expects an (H, W, 3|4) tensor; use pdum.rfb.gpu.cuda_frame() for NV12")
        height, width = int(shape[0]), int(shape[1])
        pixel_format = "rgb24" if shape[2] == 3 else "rgba8"
        data, memory = frame, "cuda"
    elif _is_metal_tensor(frame):
        shape = getattr(frame, "shape", None)
        if shape is None or len(shape) != 3 or shape[2] not in (3, 4):
            raise ValueError(
                "Metal publish expects an (H, W, 3|4) MLX array; use pdum.rfb.metal.metal_frame() for NV12"
            )
        height, width = int(shape[0]), int(shape[1])
        pixel_format = "rgb24" if shape[2] == 3 else "rgba8"
        data, memory = frame, "metal"
    else:
        raise TypeError("publish() expects a numpy.ndarray, a CUDA/Metal tensor, or a RawFrame")

    if memory == "metal":
        # Materialize the lazy MLX render on *this* (loop) thread: MLX binds a lazy graph to
        # its origin thread's stream, so the session's encode worker thread cannot evaluate a
        # frame built here. The GPU NV12 conversion still runs on the worker. See metal.materialize.
        from .metal import materialize

        materialize(data)

    if self._own_frames:
        # Server-owned mode: copy into a recycled buffer on this (loop) thread so the caller
        # may reuse/mutate its own buffer immediately. Severs the async-read aliasing entirely.
        data = self._own_copy(data, memory)

    timestamp_us = int((self._clock() - self._start) * 1_000_000)
    # seq is a placeholder; each feed stamps its own per-client sequence.
    self._latest = RawFrame(
        seq=0,
        width=width,
        height=height,
        timestamp_us=timestamp_us,
        pixel_format=pixel_format,  # type: ignore[arg-type]
        memory=memory,  # type: ignore[arg-type]
        data=data,
        pixel_ratio=resolved_pr,
        color=resolved_color,
    )
    self.width, self.height = width, height
    self._version += 1
    for feed in self._feeds:
        feed._wake()
    for tap in self._taps:
        tap._wake()

record(path, *, fps=None, bitrate=None, crf=None, preset='veryfast')

Record the published-frame stream to a real MP4 file (server-side tap).

Returns a started :class:~pdum.rfb.recording.Recording — an async context manager / handle with :meth:~pdum.rfb.recording.Recording.stop. It taps the same latest-frame stream the viewer sessions pull from, so it is independent of any connected browser and works fully headless (demos, CI artifacts, golden-frame capture). Frames are H.264-encoded (libx264) and muxed to AVCC-in-MP4 via a separate PyAV path — this is the one place AVCC/MP4 is correct; it never touches the Annex-B WebSocket payload.

The real, monotonic per-frame timestamps are honored, so a variable-cadence publisher (sparse still_after scenes included) records with correct frame timing. Call it on the event-loop thread; it starts a background encode task.

::

async with display.record("out.mp4"):
    for _ in range(300):
        display.publish(render()); await asyncio.sleep(1 / 30)
# file finalized on exit — or use rec = display.record(...); await rec.stop()

Parameters:

Name Type Description Default
path str | Path

Output .mp4 file path.

 required
fps int | None

Advisory frame rate for the encoder's rate control (default: the display's fps). The actual timing comes from the frames' real timestamps.

 None
bitrate int | None

Target bitrate in bits/s (average). Mutually exclusive with crf; if neither is given, a sensible constant-quality crf is used.

 None
crf int | None

Constant Rate Factor (0–51, lower = better) for constant-quality encoding.

 None
preset str

libx264 speed/quality preset (default "veryfast").

 'veryfast'

Raises:

Type Description
RuntimeError

If PyAV (the [h264] extra) is not installed.
Source code in src/pdum/rfb/display.py 
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
def record(
    self,
    path: str | Path,
    *,
    fps: int | None = None,
    bitrate: int | None = None,
    crf: int | None = None,
    preset: str = "veryfast",
) -> Recording:
    """Record the published-frame stream to a real **MP4** file (server-side tap).

    Returns a started :class:`~pdum.rfb.recording.Recording` — an async context
    manager / handle with :meth:`~pdum.rfb.recording.Recording.stop`. It taps the
    same latest-frame stream the viewer sessions pull from, so it is **independent
    of any connected browser** and works fully headless (demos, CI artifacts,
    golden-frame capture). Frames are H.264-encoded (libx264) and muxed to
    **AVCC-in-MP4** via a **separate** PyAV path — this is the one place AVCC/MP4 is
    correct; it never touches the Annex-B WebSocket payload.

    The real, monotonic per-frame timestamps are honored, so a variable-cadence
    publisher (sparse ``still_after`` scenes included) records with correct frame
    timing. Call it on the event-loop thread; it starts a background encode task.

    ::

        async with display.record("out.mp4"):
            for _ in range(300):
                display.publish(render()); await asyncio.sleep(1 / 30)
        # file finalized on exit — or use rec = display.record(...); await rec.stop()

    Parameters
    ----------
    path:
        Output ``.mp4`` file path.
    fps:
        Advisory frame rate for the encoder's rate control (default: the display's
        ``fps``). The *actual* timing comes from the frames' real timestamps.
    bitrate:
        Target bitrate in bits/s (average). Mutually exclusive with ``crf``; if
        neither is given, a sensible constant-quality ``crf`` is used.
    crf:
        Constant Rate Factor (0–51, lower = better) for constant-quality encoding.
    preset:
        libx264 speed/quality preset (default ``"veryfast"``).

    Raises
    ------
    RuntimeError
        If PyAV (the ``[h264]`` extra) is not installed.
    """
    from .recording import Recording

    rec = Recording(self, path, fps=fps or self.fps, bitrate=bitrate, crf=crf, preset=preset)
    rec._start_task()
    return rec

widget(*, batteries=True, base_path=None, host=None, **chrome)

Return an anywidget viewer bound to this stream (needs the [anywidget] extra).

In a notebook: display.widget() renders the batteries viewer; display.widget(batteries=False) the bare canvas. One widget = one Web Worker + one WebSocket; the server multiplexes many streams on one port. For remote/HTTPS notebooks, mount pdum.rfb.asgi.rfb_hub_endpoint same-origin and pass base_path= (the widget then uses a same-origin wss:// URL). Extra keyword args (e.g. show_toolbar=False) become widget traits.

Raises if the server is not bound yet.

Source code in src/pdum/rfb/display.py 
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
def widget(
    self,
    *,
    batteries: bool = True,
    base_path: str | None = None,
    host: str | None = None,
    **chrome: Any,
) -> Any:
    """Return an anywidget viewer bound to this stream (needs the ``[anywidget]`` extra).

    In a notebook: ``display.widget()`` renders the batteries viewer;
    ``display.widget(batteries=False)`` the bare canvas. One widget = one Web Worker +
    one WebSocket; the server multiplexes many streams on one port. For remote/HTTPS
    notebooks, mount ``pdum.rfb.asgi.rfb_hub_endpoint`` same-origin and pass
    ``base_path=`` (the widget then uses a same-origin ``wss://`` URL). Extra keyword
    args (e.g. ``show_toolbar=False``) become widget traits.

    Raises if the server is not bound yet.
    """
    from .notebook import RfbCanvas, RfbViewer

    if self.port is None:
        raise RuntimeError("Display is not serving yet; call await rfb.serve(...) first")
    resolved_host = host if host is not None else (getattr(self._owner_server, "host", None) or "127.0.0.1")
    if resolved_host in ("0.0.0.0", "", "::"):
        # Let the browser use the page's own hostname (correct for remote notebooks).
        resolved_host = "auto"
    cls = RfbViewer if batteries else RfbCanvas
    kwargs: dict[str, Any] = {"port": self.port, "stream": self._stream_name, "host": resolved_host, **chrome}
    if base_path is not None:
        kwargs["base_path"] = base_path
    return cls(**kwargs)

DownscaleHint dataclass

A server-driven adaptive resolution hint delivered through poll_events().

In the push model the publisher owns the framebuffer size, so the adaptive controller cannot resize the stream itself — its deepest congestion lever is to suggest a smaller render resolution and let the render loop honor it. Enabled by serve(adaptive=True), the controller emits one of these under sustained pressure (bitrate, fps, and in-flight already at their floors) and again on recovery (with scale climbing back toward 1.0); the :class:~pdum.rfb.display.Display fans the aggregate across viewers (the most-congested viewer wins) into the same queue :meth:~pdum.rfb.display.Display.poll_events drains, tagged as this distinct type so it is easy to tell apart from an :class:InputEvent::

for ev in display.poll_events():
    if isinstance(ev, rfb.DownscaleHint):
        w, h = ev.width, ev.height        # render/publish at this size
    else:
        state = update(state, ev.event)   # ordinary input event

Honoring the hint is opt-in: publish a frame of the suggested size (the per-viewer encoder rebuilds and the browser re-configure()s exactly as for any other resize) or ignore it entirely — the stream keeps working either way.

Parameters:

Name Type Description Default
scale float

The recommended render scale relative to full resolution, in (0, 1] (1.0 = full res). Absolute, not incremental, so applying it never compounds: round(base_w * scale) always sizes off your native resolution.

 required
width int

A convenience suggested size — the display's initial (base) dimensions scaled by scale and rounded to even values (H.264 / NV12 need even dimensions).

 required
height int

A convenience suggested size — the display's initial (base) dimensions scaled by scale and rounded to even values (H.264 / NV12 need even dimensions).

 required
received_us int

Monotonic microseconds (relative to the display's start) when the hint was emitted.

 required
Source code in src/pdum/rfb/types.py 
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
@dataclass(slots=True, frozen=True)
class DownscaleHint:
    """A server-driven adaptive **resolution** hint delivered through ``poll_events()``.

    In the push model the publisher owns the framebuffer size, so the adaptive
    controller cannot resize the stream itself — its deepest congestion lever is to
    *suggest* a smaller render resolution and let the render loop honor it. Enabled by
    ``serve(adaptive=True)``, the controller emits one of these under sustained pressure
    (bitrate, fps, and in-flight already at their floors) and again on recovery (with
    ``scale`` climbing back toward ``1.0``); the :class:`~pdum.rfb.display.Display` fans
    the aggregate across viewers (the most-congested viewer wins) into the same queue
    :meth:`~pdum.rfb.display.Display.poll_events` drains, tagged as this distinct type so
    it is easy to tell apart from an :class:`InputEvent`::

        for ev in display.poll_events():
            if isinstance(ev, rfb.DownscaleHint):
                w, h = ev.width, ev.height        # render/publish at this size
            else:
                state = update(state, ev.event)   # ordinary input event

    Honoring the hint is opt-in: publish a frame of the suggested size (the per-viewer
    encoder rebuilds and the browser re-``configure()``s exactly as for any other
    resize) or ignore it entirely — the stream keeps working either way.

    Parameters
    ----------
    scale:
        The recommended render scale relative to full resolution, in ``(0, 1]``
        (``1.0`` = full res). **Absolute**, not incremental, so applying it never
        compounds: ``round(base_w * scale)`` always sizes off your native resolution.
    width, height:
        A convenience suggested size — the display's initial (base) dimensions scaled
        by ``scale`` and rounded to even values (H.264 / NV12 need even dimensions).
    received_us:
        Monotonic microseconds (relative to the display's start) when the hint was
        emitted.
    """

    scale: float
    width: int
    height: int
    received_us: int

EncodedPayload dataclass

A single encoded payload ready to be put on the wire.

One image is one payload (always a keyframe). One encoded video access unit is one payload; keyframe marks IDR access units.

Source code in src/pdum/rfb/types.py 
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
@dataclass(slots=True)
class EncodedPayload:
    """A single encoded payload ready to be put on the wire.

    One image is one payload (always a keyframe). One encoded video access unit
    is one payload; ``keyframe`` marks IDR access units.
    """

    seq: int
    kind: EncodedKind
    timestamp_us: int
    payload: bytes
    width: int
    height: int
    mime: str | None = None  # e.g. "image/jpeg", "image/png"
    codec: str | None = None  # e.g. "avc1.42E01F"
    keyframe: bool = False
    duration_us: int | None = None
    metadata: dict[str, Any] | None = None
    # Render-side descriptors carried from the source RawFrame onto the wire header. The
    # session stamps these from the frame after encode (so no encoder needs to know about
    # them); ``header_for`` emits them when non-default. See RawFrame.pixel_ratio / color.
    pixel_ratio: float = 1.0
    color: dict | None = None

EncoderBackend

Bases: Protocol

Turns raw frames into encoded payloads.

Source code in src/pdum/rfb/types.py 
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
@runtime_checkable
class EncoderBackend(Protocol):
    """Turns raw frames into encoded payloads."""

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        """Encode a single frame, returning zero or more payloads."""
        ...

    def flush(self) -> list[EncodedPayload]:
        """Drain any buffered payloads from the encoder."""
        ...

    def close(self) -> None:
        """Release encoder resources."""
        ...

close()

Release encoder resources.

Source code in src/pdum/rfb/types.py 
231
232
233
def close(self) -> None:
    """Release encoder resources."""
    ...

encode(frame, *, force_keyframe=False)

Encode a single frame, returning zero or more payloads.

Source code in src/pdum/rfb/types.py 
223
224
225
def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
    """Encode a single frame, returning zero or more payloads."""
    ...

flush()

Drain any buffered payloads from the encoder.

Source code in src/pdum/rfb/types.py 
227
228
229
def flush(self) -> list[EncodedPayload]:
    """Drain any buffered payloads from the encoder."""
    ...

H264CpuEncoder

Encode CPU rgb24 frames to H.264 Annex B access units.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
class H264CpuEncoder:
    """Encode CPU ``rgb24`` frames to H.264 Annex B access units."""

    #: Recorded in each payload's ``metadata["encoder"]`` so the wire/headers
    #: identify which backend produced the bitstream. Subclasses override it.
    encoder_label = "h264-cpu"

    def __init__(
        self,
        *,
        width: int,
        height: int,
        fps: int = 30,
        bitrate: int = 12_000_000,
        codec_string: str | None = None,
        color: dict | None = None,
    ) -> None:
        self.width = width
        self.height = height
        self.fps = fps
        self.bitrate = bitrate
        self.codec_string = codec_string or DEFAULT_H264_CODEC
        self.color = color
        self.frame_index = 0
        self._duration_us = int(1_000_000 / fps)
        self.ctx = self._make_context()
        self._apply_color_vui(self.ctx)

    def _make_context(self):
        """Build the libx264 :class:`av.CodecContext` (overridden by NVENC)."""
        import av

        ctx = av.CodecContext.create("libx264", "w")
        ctx.width = self.width
        ctx.height = self.height
        ctx.pix_fmt = "yuv420p"
        ctx.time_base = Fraction(1, self.fps)
        ctx.framerate = Fraction(self.fps, 1)
        ctx.bit_rate = self.bitrate
        # Low latency: ultrafast, zerolatency, no B-frames, periodic in-band IDR.
        ctx.options = {
            "preset": "ultrafast",
            "tune": "zerolatency",
            "profile": "baseline",
            "forced-idr": "1",
            "x264-params": (f"keyint={self.fps}:min-keyint={self.fps}:scenecut=0:bframes=0:annexb=1:repeat-headers=1"),
        }
        return ctx

    def _apply_color_vui(self, ctx) -> None:
        """Signal color primaries/transfer/matrix/range on the codec context (→ H.264 VUI),
        shared by the libx264 and NVENC contexts. No-op for the default sRGB stream."""
        codes = h264_color_vui(self.color)
        if codes is None:
            return
        primaries, transfer, matrix, color_range = codes
        ctx.color_primaries = primaries
        ctx.color_trc = transfer
        ctx.colorspace = matrix
        ctx.color_range = color_range

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        import av

        if frame.memory == "metal":  # a published MLX frame: download to host rgb24
            from ..metal import to_host_frame

            frame = to_host_frame(frame)
        if frame.memory != "cpu" or frame.pixel_format != "rgb24":
            raise TypeError("H264CpuEncoder expects CPU rgb24 frames")
        arr = frame.data
        if not isinstance(arr, np.ndarray):
            raise TypeError("Expected numpy.ndarray")

        vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(arr), format="rgb24")
        vf = vf.reformat(format="yuv420p")
        vf.pts = self.frame_index
        vf.time_base = Fraction(1, self.fps)
        if force_keyframe:
            vf.pict_type = av.video.frame.PictureType.I
        self.frame_index += 1

        return [self._payload(frame.seq, frame.timestamp_us, pkt) for pkt in self._drain(vf)]

    def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
        """A settled-scene still for the video path: a forced **IDR** of the frame.

        True lossless H.264 isn't practical over WebCodecs, so the still here is a
        clean, self-contained intra (IDR) of the resting frame — it refreshes the
        image and lets a client that dropped deltas during a flurry jump straight
        to the latest. Re-encoding advances ``frame_index`` so the PTS stays
        monotonic. For a pixel-exact settled image, use the image transport.
        """
        return self.encode(frame, force_keyframe=True)

    def flush(self) -> list[EncodedPayload]:
        return [self._payload(-1, 0, pkt) for pkt in self._drain(None)]

    def close(self) -> None:
        try:
            self.flush()
        except Exception:  # pragma: no cover - encoder may already be closed
            pass

    # --- helpers ------------------------------------------------------------

    def _drain(self, vf):
        for packet in self.ctx.encode(vf):
            data = bytes(packet)
            if data:
                yield packet, data

    def _payload(self, seq: int, timestamp_us: int, pkt) -> EncodedPayload:
        packet, data = pkt
        return EncodedPayload(
            seq=seq,
            kind="video",
            timestamp_us=timestamp_us,
            width=self.width,
            height=self.height,
            payload=data,
            codec=self.codec_string,
            keyframe=bool(packet.is_keyframe),
            duration_us=self._duration_us,
            metadata={"bitstream": "annexb", "encoder": self.encoder_label},
        )

encode_still(frame)

A settled-scene still for the video path: a forced IDR of the frame.

True lossless H.264 isn't practical over WebCodecs, so the still here is a clean, self-contained intra (IDR) of the resting frame — it refreshes the image and lets a client that dropped deltas during a flurry jump straight to the latest. Re-encoding advances frame_index so the PTS stays monotonic. For a pixel-exact settled image, use the image transport.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
154
155
156
157
158
159
160
161
162
163
def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
    """A settled-scene still for the video path: a forced **IDR** of the frame.

    True lossless H.264 isn't practical over WebCodecs, so the still here is a
    clean, self-contained intra (IDR) of the resting frame — it refreshes the
    image and lets a client that dropped deltas during a flurry jump straight
    to the latest. Re-encoding advances ``frame_index`` so the PTS stays
    monotonic. For a pixel-exact settled image, use the image transport.
    """
    return self.encode(frame, force_keyframe=True)

ImageEncoder

Encode CPU RGB/RGBA frames to JPEG, PNG or WebP.

Source code in src/pdum/rfb/encoders/image.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
class ImageEncoder:
    """Encode CPU RGB/RGBA frames to JPEG, PNG or WebP."""

    def __init__(self, *, mode: ImageMode = "jpeg", quality: int = 80) -> None:
        self.mode: ImageMode = mode
        self.quality = quality

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        if frame.memory == "metal":  # a published MLX frame: download to host rgb24
            from ..metal import to_host_frame

            frame = to_host_frame(frame)
        if frame.memory != "cpu":
            raise TypeError("ImageEncoder expects CPU frames")

        arr = frame.data
        if not isinstance(arr, np.ndarray):
            raise TypeError("Expected numpy.ndarray")

        if frame.pixel_format == "rgb24":
            img = Image.fromarray(arr, "RGB")
        elif frame.pixel_format == "rgba8":
            img = Image.fromarray(arr, "RGBA")
        else:
            raise ValueError(f"Unsupported pixel format for image encoder: {frame.pixel_format}")

        out = BytesIO()
        if self.mode == "jpeg":
            if img.mode == "RGBA":  # JPEG cannot store alpha.
                img = img.convert("RGB")
            img.save(out, format="JPEG", quality=self.quality, optimize=False)
            mime = CAP_JPEG
        elif self.mode == "png":
            img.save(out, format="PNG")
            mime = CAP_PNG
        elif self.mode == "webp":
            img.save(out, format="WEBP", quality=self.quality)
            mime = CAP_WEBP
        else:  # pragma: no cover - guarded by the Literal type
            raise ValueError(self.mode)

        return [
            EncodedPayload(
                seq=frame.seq,
                kind="image",
                timestamp_us=frame.timestamp_us,
                width=frame.width,
                height=frame.height,
                mime=mime,
                payload=out.getvalue(),
                keyframe=True,
            )
        ]

    def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
        """Encode ``frame`` losslessly as **PNG** — the "still after settle" upgrade.

        Independent of the streaming ``mode``: once a scene settles, the resting
        frame is re-sent pixel-exact so it is crisp even when the live stream is
        lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it
        in with no client-side changes.
        """
        return ImageEncoder(mode="png").encode(frame)

    def flush(self) -> list[EncodedPayload]:
        return []

    def close(self) -> None:
        pass

encode_still(frame)

Encode frame losslessly as PNG — the "still after settle" upgrade.

Independent of the streaming mode: once a scene settles, the resting frame is re-sent pixel-exact so it is crisp even when the live stream is lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it in with no client-side changes.

Source code in src/pdum/rfb/encoders/image.py 
76
77
78
79
80
81
82
83
84
def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
    """Encode ``frame`` losslessly as **PNG** — the "still after settle" upgrade.

    Independent of the streaming ``mode``: once a scene settles, the resting
    frame is re-sent pixel-exact so it is crisp even when the live stream is
    lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it
    in with no client-side changes.
    """
    return ImageEncoder(mode="png").encode(frame)

InputEvent dataclass

A normalized user-input event delivered to the application.

Produced by :class:~pdum.rfb.display.Display as it fans connected clients' input into a single stream the application drains with :meth:~pdum.rfb.display.Display.poll_events.

Parameters:

Name Type Description Default
client_id str

Opaque per-connection identifier, so several viewers on one display can be told apart (e.g. for multi-user coordination).

 required
principal Any | None

Whatever the authenticate hook returned for this connection (an application-defined identity), or None when auth is disabled.

 required
event EventDict

The raw normalized event dict ({"type": "pointer_move", "x": ..., ...}).

 required
received_us int

Monotonic microseconds (relative to the display's start) when the event was received.

 required
Source code in src/pdum/rfb/types.py 
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
@dataclass(slots=True)
class InputEvent:
    """A normalized user-input event delivered to the application.

    Produced by :class:`~pdum.rfb.display.Display` as it fans connected clients'
    input into a single stream the application drains with
    :meth:`~pdum.rfb.display.Display.poll_events`.

    Parameters
    ----------
    client_id:
        Opaque per-connection identifier, so several viewers on one display can be
        told apart (e.g. for multi-user coordination).
    principal:
        Whatever the ``authenticate`` hook returned for this connection (an
        application-defined identity), or ``None`` when auth is disabled.
    event:
        The raw normalized event dict (``{"type": "pointer_move", "x": ..., ...}``).
    received_us:
        Monotonic microseconds (relative to the display's start) when the event
        was received.
    """

    client_id: str
    principal: Any | None
    event: EventDict
    received_us: int

NvencCpuEncoder

Bases: H264CpuEncoder

Encode CPU rgb24 frames to H.264 Annex B on the GPU via NVENC.

Drop-in for :class:H264CpuEncoder (same constructor and EncoderBackend interface); only the underlying codec context differs. Frames narrower than :data:NVENC_MIN_WIDTH are rejected because NVENC cannot open below it.

Source code in src/pdum/rfb/encoders/nvenc_cpu.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
class NvencCpuEncoder(H264CpuEncoder):
    """Encode CPU ``rgb24`` frames to H.264 Annex B on the GPU via NVENC.

    Drop-in for :class:`H264CpuEncoder` (same constructor and ``EncoderBackend``
    interface); only the underlying codec context differs. Frames narrower than
    :data:`NVENC_MIN_WIDTH` are rejected because NVENC cannot open below it.
    """

    encoder_label = "nvenc-cpu"

    def __init__(
        self,
        *,
        width: int,
        height: int,
        fps: int = 30,
        bitrate: int = 12_000_000,
        codec_string: str | None = None,
        color: dict | None = None,
    ) -> None:
        if width < NVENC_MIN_WIDTH:
            raise ValueError(
                f"NVENC requires width >= {NVENC_MIN_WIDTH}; got {width}. "
                "Use a larger framebuffer or fall back to the libx264 encoder."
            )
        super().__init__(
            width=width,
            height=height,
            fps=fps,
            bitrate=bitrate,
            codec_string=codec_string or DEFAULT_H264_CODEC,
            color=color,
        )

    def _make_context(self):
        import av

        ctx = av.CodecContext.create(_NVENC_CODEC, "w")
        ctx.width = self.width
        ctx.height = self.height
        ctx.pix_fmt = "yuv420p"
        ctx.time_base = Fraction(1, self.fps)
        ctx.framerate = Fraction(self.fps, 1)
        ctx.bit_rate = self.bitrate
        # Low latency: fast preset, low-latency tune, no B-frames, immediate
        # output, ~1 s forced-IDR cadence. ``rc=vbr`` treats ``bit_rate`` as a
        # target/ceiling and lets the stream **undershoot** on static frames —
        # critical for sparse scientific scenes (CBR would pad to the full bitrate
        # even when nothing changed, ~60x more bytes for identical quality, and
        # mirrors how the libx264 ABR path already behaves). ``profile=baseline``
        # makes the SPS advertise profile_idc 66 to match the negotiated
        # ``avc1.42E01F`` codec string (NVENC otherwise defaults to main/high).
        # NVENC emits Annex B with in-band SPS/PPS on every IDR for a raw stream.
        ctx.options = {
            "preset": "p4",
            "tune": "ll",
            "rc": "vbr",
            "profile": "baseline",
            "bf": "0",
            "delay": "0",
            "forced-idr": "1",
            "g": str(self.fps),
        }
        return ctx

NvencGpuPyavEncoder

Bases: H264CpuEncoder

Encode CUDA (or host, with upload) frames to H.264 Annex B via NVENC, zero-copy.

Drop-in for :class:~pdum.rfb.encoders.nvenc.NvencCpuEncoder (same constructor / EncoderBackend interface); only the input handling differs. Frames narrower than :data:~pdum.rfb.encoders.nvenc.NVENC_MIN_WIDTH are rejected (NVENC cannot open below it), and dimensions must be even (NV12).

Source code in src/pdum/rfb/encoders/nvenc_gpu_pyav.py 
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
class NvencGpuPyavEncoder(H264CpuEncoder):
    """Encode CUDA (or host, with upload) frames to H.264 Annex B via NVENC, zero-copy.

    Drop-in for :class:`~pdum.rfb.encoders.nvenc.NvencCpuEncoder` (same
    constructor / ``EncoderBackend`` interface); only the input handling differs.
    Frames narrower than :data:`~pdum.rfb.encoders.nvenc.NVENC_MIN_WIDTH` are
    rejected (NVENC cannot open below it), and dimensions must be even (NV12).
    """

    encoder_label = "nvenc-gpu-pyav"

    def __init__(
        self,
        *,
        width: int,
        height: int,
        fps: int = 30,
        bitrate: int = 12_000_000,
        codec_string: str | None = None,
    ) -> None:
        if width < NVENC_MIN_WIDTH:
            raise ValueError(
                f"NVENC requires width >= {NVENC_MIN_WIDTH}; got {width}. "
                "Use a larger framebuffer or fall back to the libx264 encoder."
            )
        if width % 2 or height % 2:
            raise ValueError(f"NV12 requires even dimensions; got {width}x{height}")
        # Share CuPy's primary CUDA context with FFmpeg (must precede CuPy use; a
        # no-op if the caller already did it at startup, as recommended).
        enable_cuda_context_sharing()
        self._cctx = None
        self._nv12 = None  # reusable NV12 staging buffer for the rgb/host paths
        super().__init__(
            width=width,
            height=height,
            fps=fps,
            bitrate=bitrate,
            codec_string=codec_string or DEFAULT_H264_CODEC,
        )

    def _make_context(self):
        import av
        import cupy as cp
        from av.video.frame import CudaContext

        # One persistent CUDA context shared by every from_dlpack frame and the
        # encoder, so all frames carry the same hw_frames_ctx.
        self._cctx = CudaContext(device_id=0, primary_ctx=True)
        self._nv12 = cp.empty((self.height + self.height // 2, self.width), cp.uint8)

        ctx = av.CodecContext.create(_NVENC_CODEC, "w")
        ctx.width = self.width
        ctx.height = self.height
        ctx.pix_fmt = "cuda"  # the key difference: GPU-resident input
        ctx.time_base = Fraction(1, self.fps)
        ctx.framerate = Fraction(self.fps, 1)
        ctx.bit_rate = self.bitrate
        # Same low-latency config as the host NVENC backend (see its docstring for
        # the rc=vbr / profile=baseline rationale).
        ctx.options = {
            "preset": "p4",
            "tune": "ll",
            "rc": "vbr",
            "profile": "baseline",
            "bf": "0",
            "delay": "0",
            "forced-idr": "1",
            "g": str(self.fps),
        }
        return ctx

    def _packed_nv12(self, frame):
        """Return a contiguous CUDA NV12 ``(H+H//2, W)`` buffer for ``frame``."""
        import cupy as cp

        if frame.memory == "cuda" and frame.pixel_format == "nv12":
            packed = cp.ascontiguousarray(cp.asarray(frame.data))
            if packed.shape != self._nv12.shape:
                raise ValueError(f"nv12 frame shape {packed.shape!r} != encoder {self._nv12.shape!r}")
            return packed
        if frame.pixel_format not in ("rgb24", "rgba8"):
            raise TypeError(f"NvencGpuPyavEncoder cannot encode {frame.pixel_format!r} frames")
        # rgb24/rgba8, CUDA or host: convert (uploading first if host) into the
        # reusable staging buffer. delay=0 means the prior frame is fully consumed
        # before we overwrite, so a single reused buffer is safe.
        return rgb_to_nv12(frame.data, out=self._nv12)

    def encode(self, frame, *, force_keyframe: bool = False):
        import av

        packed = self._packed_nv12(frame)
        y, uv = nv12_planes(packed)
        vf = av.VideoFrame.from_dlpack(
            [y, uv],
            format="nv12",
            width=self.width,
            height=self.height,
            primary_ctx=True,
            cuda_context=self._cctx,
        )
        vf.pts = self.frame_index
        vf.time_base = Fraction(1, self.fps)
        if force_keyframe:
            vf.pict_type = av.video.frame.PictureType.I
        self.frame_index += 1
        return [self._payload(frame.seq, frame.timestamp_us, pkt) for pkt in self._drain(vf)]

QualityTarget dataclass

A requested change to the session's encoding quality.

scale is the recommended render-resolution scale in (0, 1] (1.0 = full resolution). It is advisory to the publisher (surfaced as a :class:~pdum.rfb.types.DownscaleHint), not applied by the session like the other fields; defaults to 1.0 so existing callers are unaffected.

Source code in src/pdum/rfb/adaptive.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
@dataclass(slots=True)
class QualityTarget:
    """A requested change to the session's encoding quality.

    ``scale`` is the recommended render-resolution scale in ``(0, 1]`` (``1.0`` = full
    resolution). It is advisory to the *publisher* (surfaced as a
    :class:`~pdum.rfb.types.DownscaleHint`), not applied by the session like the other
    fields; defaults to ``1.0`` so existing callers are unaffected.
    """

    bitrate: int
    max_inflight: int
    fps: int
    scale: float = 1.0

RawFrame dataclass

A single raw frame produced by a :class:FrameSource.

Parameters:

Name Type Description Default
seq int

Monotonically increasing frame sequence number.

 required
width int

Frame dimensions in pixels.

 required
height int

Frame dimensions in pixels.

 required
timestamp_us int

Capture/render timestamp in microseconds.

 required
pixel_format PixelFormat

Layout of data (e.g. "rgb24", "rgba8", "nv12").

 required
memory MemoryKind

Where data lives ("cpu", "cuda" or "opengl").

 required
data Any

The pixel payload. For CPU frames this is a numpy.ndarray of uint8; for rgb24 the shape is (H, W, 3) and for rgba8 it is (H, W, 4).

 required
pixel_ratio float

Render-side device-pixels-per-logical-pixel of this frame (default 1.0). Display intent, not a resample instruction: the pixels are delivered as-is; the client uses it to size the frame in logical space for fit, and echoes it on input events so a publisher rendering in logical coordinates can divide it out.

 1.0
color dict | None

Optional color descriptor (dict form of a :class:ColorSpace); None means sRGB. The renderer must produce pixels in the declared space (no conversion here).

 None
Source code in src/pdum/rfb/types.py 
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
@dataclass(slots=True)
class RawFrame:
    """A single raw frame produced by a :class:`FrameSource`.

    Parameters
    ----------
    seq:
        Monotonically increasing frame sequence number.
    width, height:
        Frame dimensions in pixels.
    timestamp_us:
        Capture/render timestamp in microseconds.
    pixel_format:
        Layout of ``data`` (e.g. ``"rgb24"``, ``"rgba8"``, ``"nv12"``).
    memory:
        Where ``data`` lives (``"cpu"``, ``"cuda"`` or ``"opengl"``).
    data:
        The pixel payload. For CPU frames this is a ``numpy.ndarray`` of
        ``uint8``; for ``rgb24`` the shape is ``(H, W, 3)`` and for ``rgba8``
        it is ``(H, W, 4)``.
    pixel_ratio:
        Render-side device-pixels-per-logical-pixel of this frame (default ``1.0``).
        Display intent, not a resample instruction: the pixels are delivered as-is; the
        client uses it to size the frame in *logical* space for fit, and echoes it on
        input events so a publisher rendering in logical coordinates can divide it out.
    color:
        Optional color descriptor (``dict`` form of a :class:`ColorSpace`); ``None`` means
        sRGB. The renderer must produce pixels **in** the declared space (no conversion here).
    """

    seq: int
    width: int
    height: int
    timestamp_us: int
    pixel_format: PixelFormat
    memory: MemoryKind
    data: Any
    pixel_ratio: float = 1.0
    color: dict | None = None

Recording

Handle for an in-progress server-side MP4 recording.

Created (and started) by :meth:~pdum.rfb.display.Display.record. Use it as an async context manager (auto-finalizes on exit) or call :meth:stop explicitly::

async with display.record("out.mp4"):
    ...            # publish frames
# -- or --
rec = display.record("out.mp4")
...
await rec.stop()   # flush the encoder, finalize the container

Attributes:

Name Type Description
frames_written

Count of frames encoded into the file so far (useful for tests / progress).
Source code in src/pdum/rfb/recording.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
class Recording:
    """Handle for an in-progress server-side MP4 recording.

    Created (and started) by :meth:`~pdum.rfb.display.Display.record`. Use it as an async
    context manager (auto-finalizes on exit) or call :meth:`stop` explicitly::

        async with display.record("out.mp4"):
            ...            # publish frames
        # -- or --
        rec = display.record("out.mp4")
        ...
        await rec.stop()   # flush the encoder, finalize the container

    Attributes
    ----------
    frames_written:
        Count of frames encoded into the file so far (useful for tests / progress).
    """

    def __init__(
        self,
        display: Display,
        path: str | Path,
        *,
        fps: int,
        bitrate: int | None = None,
        crf: int | None = None,
        preset: str = "veryfast",
    ) -> None:
        if not recording_available():
            raise RuntimeError(
                "Display.record() needs PyAV (the [h264] extra). Install with pip install 'habemus-papadum-rfb[h264]'."
            )
        if bitrate is not None and crf is not None:
            raise ValueError("pass at most one of bitrate / crf")
        self._display = display
        self.path = Path(path)
        self.fps = int(fps)
        self.bitrate = bitrate
        self.crf = crf
        self.preset = preset
        self.frames_written = 0

        self._tap = _RecordingTap(display)
        self._task: asyncio.Task | None = None
        self._stopped = False
        self._error: BaseException | None = None
        # PyAV objects, created lazily on the first frame (they need its dimensions).
        self._container: Any = None
        self._stream: Any = None
        self._size: tuple[int, int] | None = None
        self._t0_us: int | None = None
        self._last_pts: int = -1

    # --- lifecycle ---------------------------------------------------------

    def _start_task(self) -> None:
        """Launch the background encode task (called by :meth:`Display.record`)."""
        self._task = asyncio.create_task(self._run())

    async def stop(self) -> None:
        """Stop recording and finalize the MP4 file (idempotent).

        Signals the tap to unpark, awaits the encode task (which flushes the encoder and
        closes the container in its ``finally``), then re-raises any error the task hit.
        """
        if self._stopped:
            return
        self._stopped = True
        self._tap.close()
        if self._task is not None:
            await self._task
            self._task = None
        if self._error is not None:
            raise self._error

    async def __aenter__(self) -> Recording:
        return self

    async def __aexit__(self, *exc: Any) -> None:
        await self.stop()

    # --- encode loop -------------------------------------------------------

    async def _run(self) -> None:
        try:
            # Open the container + libx264 context up front (off-thread), sized to the
            # display's current dimensions, so the one-time encoder init doesn't stall the
            # loop or make us drop the first frames while it warms up. Later frames of a
            # different size are scaled to fit in _write_frame.
            await asyncio.to_thread(self._ensure_container, self._display.width, self._display.height)
            while True:
                frame = await self._tap.next_frame()
                if frame is None:
                    break
                # Snapshot the (borrowed) pixels on THIS loop thread — no await between
                # next_frame() and here, so it is atomic w.r.t. a concurrent publish() —
                # then encode + mux off-thread so the event loop keeps serving viewers.
                host, av_format = _snapshot_host(frame)
                ts = int(frame.timestamp_us)
                await asyncio.to_thread(self._write_frame, host, av_format, ts)
        except BaseException as exc:  # noqa: BLE001 - stash and re-raise from stop()
            self._error = exc
        finally:
            self._display._remove_tap(self._tap)
            await asyncio.to_thread(self._finalize)

    def _ensure_container(self, width: int, height: int) -> None:
        """Open the MP4 container + libx264 stream, sized to the first frame."""
        import av

        self._container = av.open(str(self.path), mode="w")
        stream = self._container.add_stream("libx264", rate=self.fps)
        stream.width = width
        stream.height = height
        stream.pix_fmt = "yuv420p"
        stream.codec_context.time_base = _TIME_BASE
        # No B-frames (tune=zerolatency) ⇒ output DTS == PTS, monotonic — trivial to mux
        # with our real-timestamp PTS. Constant-quality by default; bitrate/crf override.
        options = {"preset": self.preset, "tune": "zerolatency"}
        if self.bitrate is not None:
            stream.bit_rate = int(self.bitrate)
        else:
            options["crf"] = str(self.crf if self.crf is not None else 20)
        stream.options = options
        self._stream = stream
        self._size = (width, height)

    def _write_frame(self, host: np.ndarray, av_format: str, ts_us: int) -> None:
        """Encode one host frame and mux it (runs in a worker thread)."""
        import av

        h, w = host.shape[0], host.shape[1]
        if self._container is None:
            self._ensure_container(w, h)
        assert self._size is not None
        vframe = av.VideoFrame.from_ndarray(host, format=av_format)
        # Fix the frame size to the stream's (scale any later resize) and go to yuv420p.
        sw, sh = self._size
        vframe = vframe.reformat(width=sw, height=sh, format="yuv420p")

        if self._t0_us is None:
            self._t0_us = ts_us
        pts = ts_us - self._t0_us
        # PTS must strictly increase (a repeated/settled timestamp would stall the muxer).
        if pts <= self._last_pts:
            pts = self._last_pts + 1
        self._last_pts = pts
        vframe.pts = pts
        vframe.time_base = _TIME_BASE

        for packet in self._stream.encode(vframe):
            self._container.mux(packet)
        self.frames_written += 1

    def _finalize(self) -> None:
        """Flush the encoder and close the container (runs in a worker thread)."""
        if self._container is None:
            return
        try:
            if self._stream is not None:
                for packet in self._stream.encode(None):  # drain buffered packets
                    self._container.mux(packet)
        finally:
            self._container.close()
            self._container = None
            self._stream = None

stop() async

Stop recording and finalize the MP4 file (idempotent).

Signals the tap to unpark, awaits the encode task (which flushes the encoder and closes the container in its finally), then re-raises any error the task hit.

Source code in src/pdum/rfb/recording.py 
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
async def stop(self) -> None:
    """Stop recording and finalize the MP4 file (idempotent).

    Signals the tap to unpark, awaits the encode task (which flushes the encoder and
    closes the container in its ``finally``), then re-raises any error the task hit.
    """
    if self._stopped:
        return
    self._stopped = True
    self._tap.close()
    if self._task is not None:
        await self._task
        self._task = None
    if self._error is not None:
        raise self._error

RfbSession

Drive one client connection: encode + send frames, receive events.

Source code in src/pdum/rfb/session.py 
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
class RfbSession:
    """Drive one client connection: encode + send frames, receive events."""

    def __init__(
        self,
        source: FrameSource,
        encoder: EncoderBackend,
        ws: Any,
        *,
        encoder_factory: EncoderFactory | None = None,
        max_inflight: int = 2,
        bitrate: int = 12_000_000,
        fps: int = 30,
        adaptive: AdaptiveQualityController | None = None,
        still_after: float | None = None,
        stats_interval: float | None = None,
        inflight_timeout: float = 2.0,
        clock: Callable[[], float] | None = None,
    ) -> None:
        self.source = source
        self.encoder = encoder
        self.ws = ws
        self.encoder_factory = encoder_factory
        self.max_inflight = max_inflight
        self.bitrate = bitrate
        self.fps = fps
        self.adaptive = adaptive
        self.still_after = still_after
        self.stats_interval = stats_interval
        # Defense-in-depth backstop (docs/proposals/completed/client_decode_resilience.md):
        # if a sent seq sits unacked longer than this, the client is wedged (a stalled
        # decoder, an old build). Clear inflight + force a keyframe rather than dropping
        # every frame forever — this breaks the client↔server deadlock from the server side.
        self.inflight_timeout = inflight_timeout
        self.inflight_timeouts = 0  # count of backstop trips (for tests / metrics)
        self.decoder_resets = 0  # count of client decoder_reset controls honored
        self._last_stats_t = -1e9
        self._clock = clock or time.monotonic

        self.force_keyframe = True
        # Adaptive resolution lever (serve(adaptive=True)): the controller's current
        # recommended render scale. The session cannot resize the shared framebuffer
        # itself, so a change is surfaced to the publisher via the source (a DownscaleHint
        # through Display.poll_events()); starts at full resolution.
        self.render_scale = 1.0
        self.inflight: set[int] = set()
        self.dropped = 0
        self._last_drop_log_t = -1e9  # rate-limit the "dropping frames" warning to ~1/s
        self.closed = False
        self._enc_size: tuple[int, int] | None = None
        self._send_times: dict[int, float] = {}
        self.metrics = SessionMetrics(started_at=self._clock(), target_bitrate=bitrate, target_fps=fps)

        # "Still after interaction settles": when the scene goes quiet (no new
        # published frame for ``still_after`` seconds), re-send the resting frame
        # at higher quality — a lossless PNG on the image path, a clean IDR on the
        # video path. Opt-in, and only when both the source can produce a still
        # and the encoder knows how to encode one (otherwise a silent no-op).
        self._still_pending = False
        self._stills_enabled = (
            still_after is not None and hasattr(encoder, "encode_still") and hasattr(source, "still_frame")
        )
        # Server-owned buffers the resting frame is snapshotted into before the off-thread
        # still encode, so a caller that reuses/mutates its published buffer while idle can't
        # corrupt the still. Allocated once and reused; reallocated only on a size/dtype change.
        # See _snapshot_still and docs/still_after_settle.md.
        self._still_buf: np.ndarray | None = None
        self._still_buf_cuda: Any = None

    def metrics_snapshot(self) -> dict:
        """Return a JSON-serializable snapshot of this session's metrics."""
        self.metrics.inflight = len(self.inflight)
        self.metrics.target_bitrate = self.bitrate
        self.metrics.target_fps = self.fps
        return self.metrics.snapshot(now=self._clock())

    # --- receive side -------------------------------------------------------

    def _reset_inflight(self) -> None:
        """Clear the unacked-payload set so the encode loop can send again, and force the next
        frame to a self-contained keyframe. Shared by the client ``decoder_reset`` control and
        the server-side inflight-timeout backstop. It does **not** ACK the stalled seqs (that
        would corrupt the displayed-FIFO seq attribution + RTT); it just stops waiting."""
        self.inflight.clear()
        self._send_times.clear()
        self.metrics.inflight = 0
        self.force_keyframe = True

    async def _handle_control(self, data: dict) -> None:
        """Process one decoded JSON control message (one step of ``recv_loop``)."""
        kind = data.get("type")
        if kind == "ack":
            seq = data.get("seq")
            now = self._clock()
            sent_at = self._send_times.pop(seq, None)
            rtt_ms = (now - sent_at) * 1000 if sent_at is not None else None
            self.inflight.discard(seq)
            self.metrics.inflight = len(self.inflight)
            self.metrics.record_ack(
                rtt_ms=rtt_ms,
                decode_queue_size=int(data.get("decode_queue_size", 0)),
                now=now,
            )
            await self._maybe_send_stats(now)
        elif kind == "request_keyframe":
            self.force_keyframe = True
        elif kind == "decoder_reset":
            # The client's decoder stalled and it rebuilt — stop waiting for frames it will
            # never display so the next encode can send it a fresh keyframe.
            self.decoder_resets += 1
            log.info("client rebuilt its decoder (stall recovery) — clearing inflight + forcing keyframe")
            self._reset_inflight()
        elif kind == "event":
            await self.source.handle_event(data["event"])
        elif kind == "set_viewport":
            # Renderview-shaped resize: logical width/height, physical pwidth/pheight,
            # ratio. Older clients sent only width/height (physical) + pixel_ratio.
            await self.source.handle_event(
                {
                    "type": "resize",
                    "width": data["width"],
                    "height": data["height"],
                    "pwidth": data.get("pwidth", data["width"]),
                    "pheight": data.get("pheight", data["height"]),
                    "ratio": data.get("ratio", data.get("pixel_ratio", 1)),
                }
            )

    async def recv_loop(self) -> None:
        try:
            async for msg in self.ws:
                if isinstance(msg, (bytes, bytearray)):
                    continue
                await self._handle_control(parse_control(msg))
        except _ConnectionClosed:
            pass
        finally:
            # When the client disconnects, stop the encode loop too.
            self.closed = True

    # --- send side ----------------------------------------------------------

    async def send_payload(self, payload: EncodedPayload) -> None:
        await self.ws.send(pack_binary_message(header_for(payload), payload.payload))
        now = self._clock()
        self.inflight.add(payload.seq)
        self._send_times[payload.seq] = now
        self.metrics.inflight = len(self.inflight)
        self.metrics.record_sent(payload_bytes=len(payload.payload), keyframe=payload.keyframe, now=now)

    #: Pending live reconfigure (backend/transport/params). Set by
    #: :meth:`request_reconfigure` and applied at the top of an encode step so it never
    #: races the off-thread ``encoder.encode``. ``None`` when idle.
    _reconfig: dict | None = None

    def request_reconfigure(
        self,
        *,
        factory: EncoderFactory | None = None,
        selection: Any = None,
        bitrate: int | None = None,
        fps: int | None = None,
    ) -> None:
        """Queue a live encoder swap and/or quality change.

        Applied before the next encode (never mid-encode). ``factory`` swaps the encoder
        backend/transport; ``selection`` (a :class:`~pdum.rfb.protocol.BackendSelection`)
        triggers a fresh ``config`` to the client; ``bitrate``/``fps`` retune quality.
        Must be called on the event-loop thread (it only stores state).
        """
        self._reconfig = {"factory": factory, "selection": selection, "bitrate": bitrate, "fps": fps}

    async def _apply_reconfigure(self, width: int, height: int) -> None:
        """Apply a pending :meth:`request_reconfigure` between encode steps."""
        req, self._reconfig = self._reconfig, None
        if req is None:
            return
        if req["factory"] is not None:
            self.encoder_factory = req["factory"]
        if req["bitrate"] is not None:
            self.bitrate = req["bitrate"]
        if req["fps"] is not None:
            self.fps = req["fps"]
        if self.encoder_factory is not None:
            self.encoder.close()
            self.encoder = self.encoder_factory(width, height, self.bitrate, self.fps)
            self._enc_size = (width, height)
            self.force_keyframe = True  # next frame is a self-contained keyframe (KeyframeGate)
        sel = req["selection"]
        if sel is not None:
            transport = "webcodecs" if sel.transport == "h264" else "image"
            await self.ws.send(config_message(transport=transport, width=width, height=height, codec=sel.codec))

    def _ensure_encoder_for(self, width: int, height: int) -> None:
        """Rebuild the (fixed-resolution) encoder if the frame size changed."""
        size = (width, height)
        if self._enc_size is None:
            self._enc_size = size
            return
        if size != self._enc_size and self.encoder_factory is not None:
            self.encoder.close()
            self.encoder = self.encoder_factory(width, height, self.bitrate, self.fps)
            self.force_keyframe = True
            self._enc_size = size

    async def _encode_step(self) -> str:
        """Run one encode iteration.

        Returns ``"sent"``, ``"dropped"``, ``"still"`` or ``"stopped"``.
        """
        try:
            frame = await self._next_frame_or_idle()
        except StopAsyncIteration:
            return "stopped"

        if frame is None:
            # The scene settled: upgrade the resting frame to a high-quality still.
            await self._send_still()
            return "still"

        # A fresh frame supersedes any still we were about to send; arm the next.
        self._still_pending = self._stills_enabled

        # Latest-frame-wins: if the client is behind, drop this frame before
        # spending encode time and force the next sent one to be a keyframe.
        if len(self.inflight) >= self.max_inflight:
            now = self._clock()
            oldest = min(self._send_times.values(), default=now)
            if self._send_times and now - oldest > self.inflight_timeout:
                # Backstop: nothing acked for inflight_timeout — the client is wedged (a
                # stalled decoder / old build). Clear inflight + force a keyframe and fall
                # through to send *this* frame, instead of dropping forever (silent deadlock).
                self.inflight_timeouts += 1
                log.warning(
                    "client unresponsive for %.1fs (wedged decoder?) — clearing inflight + forcing keyframe",
                    now - oldest,
                )
                self._reset_inflight()
            else:
                self.dropped += 1
                self.force_keyframe = True
                self.metrics.record_dropped(now=now)
                # Rate-limited so a sustained backlog logs ~once/second, not per dropped frame.
                if now - self._last_drop_log_t >= 1.0:
                    log.warning(
                        "client behind — dropping frames (%d dropped so far; next frame forced to keyframe)",
                        self.dropped,
                    )
                    self._last_drop_log_t = now
                return "dropped"

        if self._reconfig is not None:
            await self._apply_reconfigure(frame.width, frame.height)
        self._ensure_encoder_for(frame.width, frame.height)
        force = self.force_keyframe
        t0 = self._clock()
        payloads = await asyncio.to_thread(self.encoder.encode, frame, force_keyframe=force)
        self.metrics.record_encode((self._clock() - t0) * 1000, now=self._clock())
        self.force_keyframe = False

        for payload in payloads:
            self._stamp_render_descriptors(payload, frame)
            await self.send_payload(payload)
        await self._maybe_adapt()
        return "sent"

    async def _next_frame_or_idle(self) -> Any:
        """Park for the next frame; surface a settle window as ``None``.

        When stills are enabled and one is pending, the wait is bounded by
        ``still_after`` so that ``still_after`` seconds without a new published
        frame returns ``None`` ("the scene settled — send a still"). Otherwise it
        blocks until the next frame, exactly like the plain pull. The pending flag
        is cleared once a still fires, so the bounded wait happens at most once per
        settle and the loop reverts to a blocking park (no busy-loop on idle).
        """
        if not (self._stills_enabled and self._still_pending):
            return await self.source.next_frame()
        try:
            return await asyncio.wait_for(self.source.next_frame(), self.still_after)
        except TimeoutError:
            return None

    async def _send_still(self) -> None:
        """Encode and send a lossless / high-quality still of the settled frame.

        The still re-sends the *current latest* frame (the one the client is
        resting on) with a fresh per-client ``seq`` and as a self-contained
        keyframe, so a client that dropped deltas during a flurry also jumps
        straight to the latest. A one-shot nicety, so it is skipped (rather than
        queued) when the client is still catching up.
        """
        self._still_pending = False
        still = self.source.still_frame()  # type: ignore[attr-defined]
        if still is None or len(self.inflight) >= self.max_inflight:
            return
        self._ensure_encoder_for(still.width, still.height)
        # Snapshot the resting frame into a server-owned buffer on THIS (loop) thread, so the
        # off-thread encode below reads a stable copy even if the caller reuses/mutates its
        # published buffer while the scene is settled. No await between here and to_thread, so
        # the copy is atomic w.r.t. the generator (which publishes on the same thread).
        still = self._snapshot_still(still)
        t0 = self._clock()
        payloads = await asyncio.to_thread(self.encoder.encode_still, still)  # type: ignore[attr-defined]
        self.metrics.record_encode((self._clock() - t0) * 1000, now=self._clock())
        for payload in payloads:
            self._stamp_render_descriptors(payload, still)
            await self.send_payload(payload)

    @staticmethod
    def _stamp_render_descriptors(payload: EncodedPayload, frame: RawFrame) -> None:
        """Carry the source frame's render-side descriptors (``pixel_ratio`` / ``color``)
        onto the outgoing payload, so ``header_for`` can emit them without any encoder
        needing to know they exist. The session is the single frame→payload seam."""
        payload.pixel_ratio = frame.pixel_ratio
        payload.color = frame.color

    def _snapshot_still(self, frame: RawFrame) -> RawFrame:
        """Copy the resting ``frame`` into a server-owned, reused buffer for the still encode.

        Runs on the loop thread, so it is atomic w.r.t. the generator (which publishes on the
        same thread); the off-thread ``encode_still`` then reads this stable copy rather than
        the caller's buffer. The buffer is allocated once and reused; a size/dtype change
        reallocates it (mirrors the fixed-resolution encoder rebuild in ``_ensure_encoder_for``).

        Metal frames are returned unchanged: MLX arrays are functionally immutable (a render
        yields a *new* array) and ``publish()`` already materializes them on the loop thread, so
        there is no torn read to sever. See docs/still_after_settle.md.
        """
        data = frame.data
        if frame.memory == "cpu":
            arr = np.asarray(data)
            if self._still_buf is None or self._still_buf.shape != arr.shape or self._still_buf.dtype != arr.dtype:
                self._still_buf = np.empty(arr.shape, arr.dtype)  # C-contiguous, not empty_like
            np.copyto(self._still_buf, arr)
            return dataclasses.replace(frame, data=self._still_buf)
        if frame.memory == "cuda":
            import cupy as cp  # lazy: only when a cuda frame settles, so ``import pdum.rfb`` stays dep-free

            src = cp.asarray(data)
            buf = self._still_buf_cuda
            if buf is None or buf.shape != src.shape or buf.dtype != src.dtype:
                buf = self._still_buf_cuda = cp.empty(src.shape, src.dtype)
            cp.copyto(buf, src)
            return dataclasses.replace(frame, data=buf)
        return frame  # metal (MLX immutable) / other: no snapshot needed

    async def _maybe_send_stats(self, now: float) -> None:
        """Push a server-truth ``stats`` control message to the client, throttled.

        Opt-in via ``stats_interval``: lets the browser surface authoritative
        server-side metrics (RTT, fps, bitrate, encode time, the adaptive targets)
        in its ``Stats`` — the client only sees its own decode side otherwise.
        """
        if self.stats_interval is None or now - self._last_stats_t < self.stats_interval:
            return
        self._last_stats_t = now
        snap = self.metrics_snapshot()
        await self.ws.send(
            json.dumps(
                {
                    "type": "stats",
                    "rtt_ms": snap["rtt_ms"],
                    "fps_sent": snap["fps_sent"],
                    "fps_acked": snap["fps_acked"],
                    "bitrate_bps": snap["bitrate_bps"],
                    "encode_ms": snap["encode_ms"],
                    "decode_queue_size": snap["decode_queue_size"],
                    "dropped": snap["frames_dropped"],
                    "target_bitrate": snap["target_bitrate"],
                    "target_fps": snap["target_fps"],
                }
            )
        )

    async def _maybe_adapt(self) -> None:
        """Apply an adaptive-quality decision, if the controller requests one."""
        if self.adaptive is None:
            return
        target = self.adaptive.update(self.metrics_snapshot(), now=self._clock())
        if target is None:
            return
        self.max_inflight = target.max_inflight
        rebuild = target.bitrate != self.bitrate or target.fps != self.fps
        self.bitrate = target.bitrate
        self.fps = target.fps
        if rebuild and self.encoder_factory is not None and self._enc_size is not None:
            w, h = self._enc_size
            self.encoder.close()
            self.encoder = self.encoder_factory(w, h, self.bitrate, self.fps)
            self.force_keyframe = True
        # Resolution lever: the session can't resize the shared framebuffer, so hand the
        # recommendation to the publisher via the source (Display fans out a DownscaleHint).
        if target.scale != self.render_scale:
            self.render_scale = target.scale
            suggest = getattr(self.source, "suggest_render_scale", None)
            if suggest is not None:
                suggest(target.scale)
        await self.ws.send(json.dumps({"type": "set_quality", "bitrate": self.bitrate, "fps": self.fps}))

    async def encode_loop(self) -> None:
        try:
            while not self.closed:
                result = await self._encode_step()
                if result == "stopped":
                    break
                if result == "dropped":
                    await asyncio.sleep(0)
        except _ConnectionClosed:
            self.closed = True

    async def run(self) -> None:
        """Run the receive and encode loops until the connection closes."""
        try:
            async with asyncio.TaskGroup() as tg:
                tg.create_task(self.recv_loop())
                tg.create_task(self.encode_loop())
        finally:
            self.closed = True
            self.encoder.close()

metrics_snapshot()

Return a JSON-serializable snapshot of this session's metrics.

Source code in src/pdum/rfb/session.py 
113
114
115
116
117
118
def metrics_snapshot(self) -> dict:
    """Return a JSON-serializable snapshot of this session's metrics."""
    self.metrics.inflight = len(self.inflight)
    self.metrics.target_bitrate = self.bitrate
    self.metrics.target_fps = self.fps
    return self.metrics.snapshot(now=self._clock())

request_reconfigure(*, factory=None, selection=None, bitrate=None, fps=None)

Queue a live encoder swap and/or quality change.

Applied before the next encode (never mid-encode). factory swaps the encoder backend/transport; selection (a :class:~pdum.rfb.protocol.BackendSelection) triggers a fresh config to the client; bitrate/fps retune quality. Must be called on the event-loop thread (it only stores state).

Source code in src/pdum/rfb/session.py 
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
def request_reconfigure(
    self,
    *,
    factory: EncoderFactory | None = None,
    selection: Any = None,
    bitrate: int | None = None,
    fps: int | None = None,
) -> None:
    """Queue a live encoder swap and/or quality change.

    Applied before the next encode (never mid-encode). ``factory`` swaps the encoder
    backend/transport; ``selection`` (a :class:`~pdum.rfb.protocol.BackendSelection`)
    triggers a fresh ``config`` to the client; ``bitrate``/``fps`` retune quality.
    Must be called on the event-loop thread (it only stores state).
    """
    self._reconfig = {"factory": factory, "selection": selection, "bitrate": bitrate, "fps": fps}

run() async

Run the receive and encode loops until the connection closes.

Source code in src/pdum/rfb/session.py 
455
456
457
458
459
460
461
462
463
async def run(self) -> None:
    """Run the receive and encode loops until the connection closes."""
    try:
        async with asyncio.TaskGroup() as tg:
            tg.create_task(self.recv_loop())
            tg.create_task(self.encode_loop())
    finally:
        self.closed = True
        self.encoder.close()

Server

A hub: one WebSocket listener fronting several named streams.

Each stream is an independent :class:~pdum.rfb.display.Display with its own encoder config; a browser selects one by URL path (ws://host/<stream>), and a connection with no path lands on the "default" stream. Streams are discoverable over HTTP at GET /streams.

Build one with :func:serve_server, or use :func:serve for the common single-default-stream case (it returns the default Display and keeps display.server pointing back here so you can add_stream more).

This composes with everything else — multi-client fan-out, per-client backpressure, the encoders, auth, "still after settle" — none of which changes; a stream is just a Display plus its config, and routing is purely additive.

Source code in src/pdum/rfb/server.py 
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
class Server:
    """A hub: one WebSocket listener fronting several named streams.

    Each stream is an independent :class:`~pdum.rfb.display.Display` with its own
    encoder config; a browser selects one by **URL path** (``ws://host/<stream>``),
    and a connection with no path lands on the ``"default"`` stream. Streams are
    discoverable over HTTP at ``GET /streams``.

    Build one with :func:`serve_server`, or use :func:`serve` for the common
    single-default-stream case (it returns the default ``Display`` and keeps
    ``display.server`` pointing back here so you can ``add_stream`` more).

    This composes with everything else — multi-client fan-out, per-client
    backpressure, the encoders, auth, "still after settle" — none of which changes;
    a stream is just a ``Display`` plus its config, and routing is purely additive.
    """

    def __init__(
        self,
        *,
        host: str = "127.0.0.1",
        port: int = 8765,
        origins: list[str | None] | None = None,
    ) -> None:
        self.host = host
        self._port = port  # requested; the bound port may differ when port=0
        self.origins = origins
        self._streams: dict[str, _StreamHost] = {}
        self._listener: Any = None
        self._listener_cm: Any = None
        self._closed = False

    # --- streams -----------------------------------------------------------

    def add_stream(
        self,
        name: str,
        width: int,
        height: int,
        *,
        fps: int = 30,
        bitrate: int = 12_000_000,
        max_inflight: int = 2,
        has_h264: bool | None = None,
        has_nvenc: bool | None = None,
        gpu: bool = False,
        adaptive: bool = False,
        still_after: float | None = None,
        stats_interval: float | None = None,
        authenticate: Authenticator | None = None,
        record_events: bool = False,
        event_log: str | Path | None = None,
        event_queue_size: int = 4096,
        own_frames: bool = False,
        resize_policy: str = "publisher",
        max_render_dimension: int | None = None,
        encode_pipeline_depth: int = 0,
    ) -> Display:
        """Register a new named stream and return its :class:`Display`.

        Streams are independent: each carries its own encoder config (one GPU, one
        image; per-stream bitrate; per-stream ``authenticate``). Safe to call before
        or after :meth:`start` — clients reach it at ``ws://host/<name>`` either way.
        Raises if ``name`` is already taken. ``own_frames`` / ``resize_policy`` /
        ``max_render_dimension`` are forwarded to the stream's :class:`Display`.
        """
        if name in self._streams:
            raise ValueError(f"stream {name!r} already exists")
        display = Display(
            width,
            height,
            fps=fps,
            record_events=record_events,
            event_log=event_log,
            event_queue_size=event_queue_size,
            own_frames=own_frames,
            resize_policy=resize_policy,
            max_render_dimension=max_render_dimension,
        )
        host = _StreamHost(
            display,
            name,
            has_h264=has_h264,
            has_nvenc=has_nvenc,
            fps=fps,
            bitrate=bitrate,
            max_inflight=max_inflight,
            adaptive=adaptive,
            still_after=still_after,
            stats_interval=stats_interval,
            authenticate=authenticate,
            gpu=gpu,
            encode_pipeline_depth=encode_pipeline_depth,
        )
        self._streams[name] = host
        display._owner_server = self
        display._stream_name = name
        display._server = self._listener  # None until start(); back-filled there
        return display

    def stream(self, name: str = DEFAULT_STREAM) -> Display:
        """Return the :class:`Display` for stream ``name`` (``KeyError`` if absent)."""
        return self._streams[name].display

    def remove_stream(self, name: str) -> None:
        """Remove a named stream, disconnecting its viewers — the inverse of :meth:`add_stream`.

        No-op if ``name`` is absent. The stream's :class:`Display` is closed locally
        (viewers dropped, waiters woken) but the shared listener keeps running for the
        other streams; a later connection to ``name`` closes with ``4404``. Used by the
        demo hub to reap idle per-client streams.
        """
        host = self._streams.pop(name, None)
        if host is not None:
            host.display._close_local()

    @property
    def streams(self) -> list[str]:
        """The names of the registered streams."""
        return list(self._streams)

    @property
    def port(self) -> int | None:
        """The bound TCP port (the actual one when started with ``port=0``)."""
        if self._listener is not None:
            return next(iter(self._listener.sockets)).getsockname()[1]
        return self._port

    # --- lifecycle ---------------------------------------------------------

    async def start(self) -> Server:
        """Start the shared listener in the background; returns ``self``."""
        import websockets.asyncio.server

        kwargs: dict[str, Any] = dict(process_request=self.process_request, max_size=None)
        if self.origins is not None:
            kwargs["origins"] = self.origins
        cm = websockets.asyncio.server.serve(self._route, self.host, self._port, **kwargs)
        self._listener = await cm.__aenter__()
        self._listener_cm = cm
        for host in self._streams.values():
            host.display._server = self._listener
        return self

    async def aclose(self) -> None:
        """Stop the listener and disconnect every viewer of every stream."""
        if self._closed:
            return
        self._closed = True
        if self._listener_cm is not None:
            cm, self._listener_cm = self._listener_cm, None
            self._listener = None
            await cm.__aexit__(None, None, None)
        for host in self._streams.values():
            host.display._close_local()

    async def __aenter__(self) -> Server:
        return await self.start()

    async def __aexit__(self, *exc: Any) -> None:
        await self.aclose()

    # --- routing -----------------------------------------------------------

    @staticmethod
    def _stream_name(path: str) -> str:
        """Map a request path to a stream name (first path segment; ``"default"``)."""
        seg = path.split("?", 1)[0].strip("/").split("/", 1)[0]
        return seg or DEFAULT_STREAM

    async def _route(self, connection: Any) -> None:
        """Dispatch one connection to its stream by URL path (close 4404 if unknown)."""
        req = getattr(connection, "request", None)
        name = self._stream_name(getattr(req, "path", "") or "")
        host = self._streams.get(name)
        if host is None:
            await connection.close(4404, f"unknown stream {name!r}")
            return
        await host.handler(connection)

    def process_request(self, connection: Any, request: Any):
        """Answer the HTTP side-channel routes; return None to proceed with WS.

        Global: ``GET /health``, ``GET /streams``, ``GET /streams/<name>/metrics``.
        For backward compatibility the single-stream routes (``/metrics``,
        ``/recorded-events``, ``/recorded-events/reset``) act on the ``"default"``
        stream when one exists.
        """
        path = request.path.split("?", 1)[0]
        if path == "/health":
            return connection.respond(HTTPStatus.OK, "ok\n")
        if path == "/streams":
            return connection.respond(HTTPStatus.OK, json.dumps([h.info() for h in self._streams.values()]))
        parts = path.strip("/").split("/")
        if len(parts) == 3 and parts[0] == "streams" and parts[2] == "metrics":
            host = self._streams.get(parts[1])
            if host is None:
                return connection.respond(HTTPStatus.NOT_FOUND, "[]")
            return connection.respond(HTTPStatus.OK, json.dumps(host.metrics()))
        default = self._streams.get(DEFAULT_STREAM)
        if default is not None:
            if path == "/metrics":
                return connection.respond(HTTPStatus.OK, json.dumps(default.metrics()))
            if path == "/recorded-events":
                return connection.respond(HTTPStatus.OK, json.dumps(default.display.recorded))
            if path == "/recorded-events/reset":
                default.display.recorded.clear()
                return connection.respond(HTTPStatus.OK, "[]")
        return None

port property

The bound TCP port (the actual one when started with port=0).

streams property

The names of the registered streams.

aclose() async

Stop the listener and disconnect every viewer of every stream.

Source code in src/pdum/rfb/server.py 
571
572
573
574
575
576
577
578
579
580
581
async def aclose(self) -> None:
    """Stop the listener and disconnect every viewer of every stream."""
    if self._closed:
        return
    self._closed = True
    if self._listener_cm is not None:
        cm, self._listener_cm = self._listener_cm, None
        self._listener = None
        await cm.__aexit__(None, None, None)
    for host in self._streams.values():
        host.display._close_local()

add_stream(name, width, height, *, fps=30, bitrate=12000000, max_inflight=2, has_h264=None, has_nvenc=None, gpu=False, adaptive=False, still_after=None, stats_interval=None, authenticate=None, record_events=False, event_log=None, event_queue_size=4096, own_frames=False, resize_policy='publisher', max_render_dimension=None, encode_pipeline_depth=0)

Register a new named stream and return its :class:Display.

Streams are independent: each carries its own encoder config (one GPU, one image; per-stream bitrate; per-stream authenticate). Safe to call before or after :meth:start — clients reach it at ws://host/<name> either way. Raises if name is already taken. own_frames / resize_policy / max_render_dimension are forwarded to the stream's :class:Display.

Source code in src/pdum/rfb/server.py 
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
def add_stream(
    self,
    name: str,
    width: int,
    height: int,
    *,
    fps: int = 30,
    bitrate: int = 12_000_000,
    max_inflight: int = 2,
    has_h264: bool | None = None,
    has_nvenc: bool | None = None,
    gpu: bool = False,
    adaptive: bool = False,
    still_after: float | None = None,
    stats_interval: float | None = None,
    authenticate: Authenticator | None = None,
    record_events: bool = False,
    event_log: str | Path | None = None,
    event_queue_size: int = 4096,
    own_frames: bool = False,
    resize_policy: str = "publisher",
    max_render_dimension: int | None = None,
    encode_pipeline_depth: int = 0,
) -> Display:
    """Register a new named stream and return its :class:`Display`.

    Streams are independent: each carries its own encoder config (one GPU, one
    image; per-stream bitrate; per-stream ``authenticate``). Safe to call before
    or after :meth:`start` — clients reach it at ``ws://host/<name>`` either way.
    Raises if ``name`` is already taken. ``own_frames`` / ``resize_policy`` /
    ``max_render_dimension`` are forwarded to the stream's :class:`Display`.
    """
    if name in self._streams:
        raise ValueError(f"stream {name!r} already exists")
    display = Display(
        width,
        height,
        fps=fps,
        record_events=record_events,
        event_log=event_log,
        event_queue_size=event_queue_size,
        own_frames=own_frames,
        resize_policy=resize_policy,
        max_render_dimension=max_render_dimension,
    )
    host = _StreamHost(
        display,
        name,
        has_h264=has_h264,
        has_nvenc=has_nvenc,
        fps=fps,
        bitrate=bitrate,
        max_inflight=max_inflight,
        adaptive=adaptive,
        still_after=still_after,
        stats_interval=stats_interval,
        authenticate=authenticate,
        gpu=gpu,
        encode_pipeline_depth=encode_pipeline_depth,
    )
    self._streams[name] = host
    display._owner_server = self
    display._stream_name = name
    display._server = self._listener  # None until start(); back-filled there
    return display

process_request(connection, request)

Answer the HTTP side-channel routes; return None to proceed with WS.

Global: GET /health, GET /streams, GET /streams/<name>/metrics. For backward compatibility the single-stream routes (/metrics, /recorded-events, /recorded-events/reset) act on the "default" stream when one exists.

Source code in src/pdum/rfb/server.py 
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
def process_request(self, connection: Any, request: Any):
    """Answer the HTTP side-channel routes; return None to proceed with WS.

    Global: ``GET /health``, ``GET /streams``, ``GET /streams/<name>/metrics``.
    For backward compatibility the single-stream routes (``/metrics``,
    ``/recorded-events``, ``/recorded-events/reset``) act on the ``"default"``
    stream when one exists.
    """
    path = request.path.split("?", 1)[0]
    if path == "/health":
        return connection.respond(HTTPStatus.OK, "ok\n")
    if path == "/streams":
        return connection.respond(HTTPStatus.OK, json.dumps([h.info() for h in self._streams.values()]))
    parts = path.strip("/").split("/")
    if len(parts) == 3 and parts[0] == "streams" and parts[2] == "metrics":
        host = self._streams.get(parts[1])
        if host is None:
            return connection.respond(HTTPStatus.NOT_FOUND, "[]")
        return connection.respond(HTTPStatus.OK, json.dumps(host.metrics()))
    default = self._streams.get(DEFAULT_STREAM)
    if default is not None:
        if path == "/metrics":
            return connection.respond(HTTPStatus.OK, json.dumps(default.metrics()))
        if path == "/recorded-events":
            return connection.respond(HTTPStatus.OK, json.dumps(default.display.recorded))
        if path == "/recorded-events/reset":
            default.display.recorded.clear()
            return connection.respond(HTTPStatus.OK, "[]")
    return None

remove_stream(name)

Remove a named stream, disconnecting its viewers — the inverse of :meth:add_stream.

No-op if name is absent. The stream's :class:Display is closed locally (viewers dropped, waiters woken) but the shared listener keeps running for the other streams; a later connection to name closes with 4404. Used by the demo hub to reap idle per-client streams.

Source code in src/pdum/rfb/server.py 
531
532
533
534
535
536
537
538
539
540
541
def remove_stream(self, name: str) -> None:
    """Remove a named stream, disconnecting its viewers — the inverse of :meth:`add_stream`.

    No-op if ``name`` is absent. The stream's :class:`Display` is closed locally
    (viewers dropped, waiters woken) but the shared listener keeps running for the
    other streams; a later connection to ``name`` closes with ``4404``. Used by the
    demo hub to reap idle per-client streams.
    """
    host = self._streams.pop(name, None)
    if host is not None:
        host.display._close_local()

start() async

Start the shared listener in the background; returns self.

Source code in src/pdum/rfb/server.py 
557
558
559
560
561
562
563
564
565
566
567
568
569
async def start(self) -> Server:
    """Start the shared listener in the background; returns ``self``."""
    import websockets.asyncio.server

    kwargs: dict[str, Any] = dict(process_request=self.process_request, max_size=None)
    if self.origins is not None:
        kwargs["origins"] = self.origins
    cm = websockets.asyncio.server.serve(self._route, self.host, self._port, **kwargs)
    self._listener = await cm.__aenter__()
    self._listener_cm = cm
    for host in self._streams.values():
        host.display._server = self._listener
    return self

stream(name=DEFAULT_STREAM)

Return the :class:Display for stream name (KeyError if absent).

Source code in src/pdum/rfb/server.py 
527
528
529
def stream(self, name: str = DEFAULT_STREAM) -> Display:
    """Return the :class:`Display` for stream ``name`` (``KeyError`` if absent)."""
    return self._streams[name].display

SessionMetrics dataclass

Mutable accumulator of one session's performance counters.

Source code in src/pdum/rfb/metrics.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
@dataclass(slots=True)
class SessionMetrics:
    """Mutable accumulator of one session's performance counters."""

    started_at: float = 0.0
    updated_at: float = 0.0

    frames_sent: int = 0
    frames_dropped: int = 0
    frames_acked: int = 0
    keyframes_sent: int = 0
    bytes_sent: int = 0

    encode_ms: float = 0.0  # EMA of encoder.encode() wall time
    rtt_ms: float = 0.0  # EMA of send -> displayed-ack round trip
    decode_queue_size: int = 0  # last value reported by the client

    # Reflected from the session so a snapshot is self-contained.
    inflight: int = 0
    target_bitrate: int = 0
    target_fps: int = 0

    _extra: dict = field(default_factory=dict)
    # Rolling windows of recent event timestamps for windowed rates. `_sent_window`
    # holds (t, payload_bytes) per sent frame; `_acked_window` holds ack timestamps.
    _sent_window: deque[tuple[float, int]] = field(default_factory=deque)
    _acked_window: deque[float] = field(default_factory=deque)

    def record_encode(self, ms: float, *, now: float) -> None:
        self.encode_ms = _ema(self.encode_ms, ms)
        self.updated_at = now

    def record_sent(self, *, payload_bytes: int, keyframe: bool, now: float) -> None:
        self.frames_sent += 1
        self.bytes_sent += payload_bytes
        if keyframe:
            self.keyframes_sent += 1
        self._sent_window.append((now, payload_bytes))
        self.updated_at = now

    def record_dropped(self, *, now: float) -> None:
        self.frames_dropped += 1
        self.updated_at = now

    def record_ack(self, *, rtt_ms: float | None, decode_queue_size: int, now: float) -> None:
        self.frames_acked += 1
        if rtt_ms is not None:
            self.rtt_ms = _ema(self.rtt_ms, rtt_ms)
        self.decode_queue_size = decode_queue_size
        self._acked_window.append(now)
        self.updated_at = now

    def snapshot(self, *, now: float) -> dict:
        """Return a JSON-serializable view including derived rates."""
        elapsed = max(1e-6, now - self.started_at)
        # Windowed throughput: only the last `_RATE_WINDOW_S` of activity. The span
        # is the window unless the session is younger, so early rates aren't diluted.
        cutoff = now - _RATE_WINDOW_S
        while self._sent_window and self._sent_window[0][0] < cutoff:
            self._sent_window.popleft()
        while self._acked_window and self._acked_window[0] < cutoff:
            self._acked_window.popleft()
        span = min(_RATE_WINDOW_S, elapsed)
        window_bytes = sum(b for _, b in self._sent_window)
        return {
            "elapsed_s": round(elapsed, 3),
            "frames_sent": self.frames_sent,
            "frames_dropped": self.frames_dropped,
            "frames_acked": self.frames_acked,
            "keyframes_sent": self.keyframes_sent,
            "bytes_sent": self.bytes_sent,
            "fps_sent": round(len(self._sent_window) / span, 2),
            "fps_acked": round(len(self._acked_window) / span, 2),
            "bitrate_bps": round(window_bytes * 8 / span),
            "encode_ms": round(self.encode_ms, 2),
            "rtt_ms": round(self.rtt_ms, 2),
            "decode_queue_size": self.decode_queue_size,
            "inflight": self.inflight,
            "target_bitrate": self.target_bitrate,
            "target_fps": self.target_fps,
        }

snapshot(*, now)

Return a JSON-serializable view including derived rates.

Source code in src/pdum/rfb/metrics.py 
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
def snapshot(self, *, now: float) -> dict:
    """Return a JSON-serializable view including derived rates."""
    elapsed = max(1e-6, now - self.started_at)
    # Windowed throughput: only the last `_RATE_WINDOW_S` of activity. The span
    # is the window unless the session is younger, so early rates aren't diluted.
    cutoff = now - _RATE_WINDOW_S
    while self._sent_window and self._sent_window[0][0] < cutoff:
        self._sent_window.popleft()
    while self._acked_window and self._acked_window[0] < cutoff:
        self._acked_window.popleft()
    span = min(_RATE_WINDOW_S, elapsed)
    window_bytes = sum(b for _, b in self._sent_window)
    return {
        "elapsed_s": round(elapsed, 3),
        "frames_sent": self.frames_sent,
        "frames_dropped": self.frames_dropped,
        "frames_acked": self.frames_acked,
        "keyframes_sent": self.keyframes_sent,
        "bytes_sent": self.bytes_sent,
        "fps_sent": round(len(self._sent_window) / span, 2),
        "fps_acked": round(len(self._acked_window) / span, 2),
        "bitrate_bps": round(window_bytes * 8 / span),
        "encode_ms": round(self.encode_ms, 2),
        "rtt_ms": round(self.rtt_ms, 2),
        "decode_queue_size": self.decode_queue_size,
        "inflight": self.inflight,
        "target_bitrate": self.target_bitrate,
        "target_fps": self.target_fps,
    }

UnsupportedClient

Bases: Exception

Raised when a client advertises no transport the server can satisfy.

Source code in src/pdum/rfb/protocol.py 
43
44
class UnsupportedClient(Exception):
    """Raised when a client advertises no transport the server can satisfy."""

WebSocketTransport

Adapt a websockets server connection to the :class:Channel surface.

A raw websockets connection already satisfies :class:Channel; this thin wrapper exists as the documented seam (and one place to translate disconnect semantics for non-websockets transports later).

Source code in src/pdum/rfb/transport.py 
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
class WebSocketTransport:
    """Adapt a ``websockets`` server connection to the :class:`Channel` surface.

    A raw ``websockets`` connection already satisfies :class:`Channel`; this thin
    wrapper exists as the documented seam (and one place to translate disconnect
    semantics for non-``websockets`` transports later).
    """

    __slots__ = ("_ws",)

    def __init__(self, ws: Any) -> None:
        self._ws = ws

    async def send(self, data: bytes | str) -> None:
        await self._ws.send(data)

    def __aiter__(self) -> AsyncIterator[bytes | str]:
        return self._ws.__aiter__()

    async def close(self, code: int = 1000, reason: str = "") -> None:
        await self._ws.close(code, reason)

__getattr__(name)

Lazily import optional / submodule-executing symbols (PEP 562).

server is imported lazily so python -m pdum.rfb.server does not warn about double execution; the H.264 symbols are lazy so base installs without the optional av dependency can still import pdum.rfb.

Source code in src/pdum/rfb/__init__.py 
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
def __getattr__(name: str):
    """Lazily import optional / submodule-executing symbols (PEP 562).

    ``server`` is imported lazily so ``python -m pdum.rfb.server`` does not warn
    about double execution; the H.264 symbols are lazy so base installs without
    the optional ``av`` dependency can still ``import pdum.rfb``.
    """
    if name in ("H264CpuEncoder", "h264_available"):
        from .encoders import h264_cpu

        return getattr(h264_cpu, name)
    if name in ("NvencCpuEncoder", "nvenc_cpu_available"):
        from .encoders import nvenc_cpu

        return getattr(nvenc_cpu, name)
    if name in ("NvencGpuPyavEncoder", "nvenc_gpu_pyav_available"):
        from .encoders import nvenc_gpu_pyav

        return getattr(nvenc_gpu_pyav, name)
    if name in ("serve", "serve_server", "Server"):
        from . import server

        return getattr(server, name)
    if name == "Recording":
        from .recording import Recording

        return Recording
    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

available_video_encoders()

Return the names of registered video encoders.

Source code in src/pdum/rfb/encoders/base.py 
29
30
31
def available_video_encoders() -> list[str]:
    """Return the names of registered video encoders."""
    return sorted(_VIDEO_ENCODERS)

build_encoder(selection, *, width, height, fps=30, bitrate=12000000, video_encoder='h264_cpu', pipeline_depth=0, color=None)

Build the encoder backend described by selection.

Parameters:

Name Type Description Default
selection BackendSelection

The result of :func:pdum.rfb.protocol.select_transport.

 required
width int

Encoder configuration (ignored by the image encoder except where noted).

 required
height int

Encoder configuration (ignored by the image encoder except where noted).

 required
fps int

Encoder configuration (ignored by the image encoder except where noted).

 required
bitrate int

Encoder configuration (ignored by the image encoder except where noted).

 required
video_encoder str

Which registered video encoder to use for the H.264 transport.

 'h264_cpu'
pipeline_depth int

Encoder pipeline depth. 0 (default) is synchronous 1-in-1-out (lowest latency, seq attribution trivially correct). > 0 opts into the token-based pipelined path on backends that implement it (NVENC; on VideoToolbox it is correct but not faster). See :doc:pipelined_encode.

 0
color dict | None

Optional stream color descriptor (dict form of a :class:~pdum.rfb.types.ColorSpace). The PyAV H.264 backends tag the bitstream VUI with it; the GPU-SDK / VideoToolbox backends currently ignore it (native follow-up).

 None
Source code in src/pdum/rfb/encoders/base.py 
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def build_encoder(
    selection: BackendSelection,
    *,
    width: int,
    height: int,
    fps: int = 30,
    bitrate: int = 12_000_000,
    video_encoder: str = "h264_cpu",
    pipeline_depth: int = 0,
    color: dict | None = None,
) -> EncoderBackend:
    """Build the encoder backend described by ``selection``.

    Parameters
    ----------
    selection:
        The result of :func:`pdum.rfb.protocol.select_transport`.
    width, height, fps, bitrate:
        Encoder configuration (ignored by the image encoder except where noted).
    video_encoder:
        Which registered video encoder to use for the H.264 transport.
    pipeline_depth:
        Encoder pipeline depth. ``0`` (default) is synchronous 1-in-1-out (lowest latency,
        seq attribution trivially correct). ``> 0`` opts into the token-based pipelined path
        on backends that implement it (NVENC; on VideoToolbox it is correct but not faster).
        See :doc:`pipelined_encode`.
    color:
        Optional stream color descriptor (``dict`` form of a
        :class:`~pdum.rfb.types.ColorSpace`). The PyAV H.264 backends tag the bitstream VUI
        with it; the GPU-SDK / VideoToolbox backends currently ignore it (native follow-up).
    """
    if selection.transport == "image":
        return ImageEncoder(mode=selection.image_mode or "jpeg")

    if selection.transport == "h264":
        try:
            factory = _VIDEO_ENCODERS[video_encoder]
        except KeyError as exc:
            raise ValueError(
                f"unknown video encoder {video_encoder!r}; available: {available_video_encoders()}"
            ) from exc
        return factory(
            width=width,
            height=height,
            fps=fps,
            bitrate=bitrate,
            codec_string=selection.codec,
            pipeline_depth=pipeline_depth,
            color=color,
        )

    raise ValueError(f"unsupported transport: {selection.transport!r}")

cuda_frame(array, *, pixel_format='auto', width=None, height=None, seq=0, timestamp_us=0)

Wrap a device tensor as a CUDA :class:~pdum.rfb.types.RawFrame for publish().

pixel_format="auto" infers from shape: (H, W, 3) -> rgb24, (H, W, 4) -> rgba8, 2-D (H+H//2, W) -> nv12. Pass an explicit pixel_format (and height for ambiguous NV12) to override. The tensor is referenced, not copied; keep it alive until the frame is encoded.

Source code in src/pdum/rfb/gpu.py 
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
def cuda_frame(
    array: Any,
    *,
    pixel_format: str = "auto",
    width: int | None = None,
    height: int | None = None,
    seq: int = 0,
    timestamp_us: int = 0,
) -> RawFrame:
    """Wrap a device tensor as a CUDA :class:`~pdum.rfb.types.RawFrame` for ``publish()``.

    ``pixel_format="auto"`` infers from shape: ``(H, W, 3)`` -> ``rgb24``,
    ``(H, W, 4)`` -> ``rgba8``, 2-D ``(H+H//2, W)`` -> ``nv12``. Pass an explicit
    ``pixel_format`` (and ``height`` for ambiguous NV12) to override. The tensor
    is referenced, not copied; keep it alive until the frame is encoded.
    """
    arr = _as_cupy(array)
    if pixel_format == "auto":
        if arr.ndim == 3 and arr.shape[2] == 3:
            pixel_format = "rgb24"
        elif arr.ndim == 3 and arr.shape[2] == 4:
            pixel_format = "rgba8"
        elif arr.ndim == 2:
            pixel_format = "nv12"
        else:
            raise ValueError(f"cannot infer pixel_format from shape {arr.shape!r}")
    if pixel_format == "nv12":
        width = int(arr.shape[1]) if width is None else width
        height = nv12_height(arr) if height is None else height
    else:
        height = int(arr.shape[0]) if height is None else height
        width = int(arr.shape[1]) if width is None else width
    return RawFrame(
        seq=seq,
        width=int(width),
        height=int(height),
        timestamp_us=int(timestamp_us),
        pixel_format=pixel_format,  # type: ignore[arg-type]
        memory="cuda",
        data=arr,
    )

cuda_zerocopy_available() cached

True if the zero-copy CUDA→NVENC path is usable in this process (cached).

Checks, in order: CuPy importable; an NVENC-capable GPU/driver (:func:pdum.rfb.encoders.nvenc_cpu.nvenc_cpu_available); and that PyAV can actually encode a CUDA frame (PyAV ≥ 18 or a from-source build with the fix). The self-test opens an NVENC session, so it runs at most once per process.

.. note:: Call :func:enable_cuda_context_sharing before any CuPy CUDA op for a reliable result — if CuPy has already activated the primary context with the default flags, the shared-context probe (and the encoder) will fail.

Source code in src/pdum/rfb/gpu.py 
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
@functools.lru_cache(maxsize=1)
def cuda_zerocopy_available() -> bool:
    """True if the zero-copy CUDA→NVENC path is usable in this process (cached).

    Checks, in order: CuPy importable; an NVENC-capable GPU/driver
    (:func:`pdum.rfb.encoders.nvenc_cpu.nvenc_cpu_available`); and that PyAV can actually
    encode a CUDA frame (PyAV ≥ 18 or a from-source build with the fix). The
    self-test opens an NVENC session, so it runs at most once per process.

    .. note::
       Call :func:`enable_cuda_context_sharing` **before any CuPy CUDA op** for a
       reliable result — if CuPy has already activated the primary context with
       the default flags, the shared-context probe (and the encoder) will fail.
    """
    if importlib.util.find_spec("cupy") is None:
        return False
    try:
        from .encoders.nvenc_cpu import nvenc_cpu_available

        if not nvenc_cpu_available():
            return False
    except Exception:
        return False
    return _selftest_zerocopy_encode()

enable_cuda_context_sharing(device_id=0, *, sched=_FFMPEG_SCHED)

Pre-set the device primary-context flags so CuPy and FFmpeg share one context.

Call this once, first thing in your program, before any CuPy/PyTorch CUDA op. It sets the device's primary-context scheduling flags to what FFmpeg's CUDA hwcontext wants (CU_CTX_SCHED_BLOCKING_SYNC); CuPy then retains that same primary context, and the zero-copy encoder (primary_ctx=True) can register CuPy's device pointers.

Returns True on success. Returns False (and changes nothing) if the CUDA driver can't be loaded. If the primary context is already active with different flags (e.g. CuPy already ran), the driver call may still succeed but the flags will not take effect until the context is reset — so order matters.

Parameters:

Name Type Description Default
device_id int

CUDA device ordinal.

 0
sched str

One of "blocking_sync" (default, required for FFmpeg sharing), "spin", "yield", "auto".

 _FFMPEG_SCHED
Source code in src/pdum/rfb/gpu.py 
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
def enable_cuda_context_sharing(device_id: int = 0, *, sched: str = _FFMPEG_SCHED) -> bool:
    """Pre-set the device primary-context flags so CuPy and FFmpeg share one context.

    **Call this once, first thing in your program, before any CuPy/PyTorch CUDA
    op.** It sets the device's primary-context scheduling flags to what FFmpeg's
    CUDA hwcontext wants (``CU_CTX_SCHED_BLOCKING_SYNC``); CuPy then retains that
    same primary context, and the zero-copy encoder (``primary_ctx=True``) can
    register CuPy's device pointers.

    Returns ``True`` on success. Returns ``False`` (and changes nothing) if the
    CUDA driver can't be loaded. If the primary context is *already active* with
    different flags (e.g. CuPy already ran), the driver call may still succeed but
    the flags will not take effect until the context is reset — so order matters.

    Parameters
    ----------
    device_id:
        CUDA device ordinal.
    sched:
        One of ``"blocking_sync"`` (default, required for FFmpeg sharing),
        ``"spin"``, ``"yield"``, ``"auto"``.
    """
    cu = _libcuda()
    if cu is None:
        return False
    flag = _SCHED_FLAGS[sched]
    if cu.cuInit(0) != 0:
        return False
    dev = ctypes.c_int()
    if cu.cuDeviceGet(ctypes.byref(dev), device_id) != 0:
        return False
    return cu.cuDevicePrimaryCtxSetFlags(dev, flag) == 0

h264_available()

True if PyAV is importable.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
29
30
31
def h264_available() -> bool:
    """True if PyAV is importable."""
    return importlib.util.find_spec("av") is not None

metal_frame(array, *, pixel_format='auto', width=None, height=None, seq=0, timestamp_us=0)

Wrap an MLX (Metal) array as a memory="metal" :class:~pdum.rfb.types.RawFrame for publish() — the Metal analog of :func:pdum.rfb.gpu.cuda_frame.

pixel_format="auto" infers from shape: (H, W, 3) → rgb24, (H, W, 4) → rgba8, 2-D (H+H//2, W) → nv12. Use this for a pre-converted NV12 array; a plain RGBA array can be handed straight to display.publish() (it is recognized as a Metal frame). The array is referenced, not copied; keep it alive (and evaluated) until it is encoded.

Source code in src/pdum/rfb/metal.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def metal_frame(
    array: Any,
    *,
    pixel_format: str = "auto",
    width: int | None = None,
    height: int | None = None,
    seq: int = 0,
    timestamp_us: int = 0,
) -> RawFrame:
    """Wrap an MLX (Metal) array as a ``memory="metal"`` :class:`~pdum.rfb.types.RawFrame` for
    ``publish()`` — the Metal analog of :func:`pdum.rfb.gpu.cuda_frame`.

    ``pixel_format="auto"`` infers from shape: ``(H, W, 3)`` → ``rgb24``, ``(H, W, 4)`` → ``rgba8``,
    2-D ``(H+H//2, W)`` → ``nv12``. Use this for a **pre-converted NV12** array; a plain RGBA array
    can be handed straight to ``display.publish()`` (it is recognized as a Metal frame). The array
    is referenced, not copied; keep it alive (and evaluated) until it is encoded.
    """
    import mlx.core as mx

    if not isinstance(array, mx.array):
        array = mx.array(array)
    shape = tuple(int(s) for s in array.shape)
    if pixel_format == "auto":
        if len(shape) == 3 and shape[2] == 3:
            pixel_format = "rgb24"
        elif len(shape) == 3 and shape[2] == 4:
            pixel_format = "rgba8"
        elif len(shape) == 2:
            pixel_format = "nv12"
        else:
            raise ValueError(f"cannot infer pixel_format from shape {shape!r}")
    if pixel_format == "nv12":
        width = shape[1] if width is None else width
        height = (shape[0] * 2 // 3) if height is None else height
    else:
        height = shape[0] if height is None else height
        width = shape[1] if width is None else width
    return RawFrame(
        seq=seq,
        width=int(width),
        height=int(height),
        timestamp_us=int(timestamp_us),
        pixel_format=pixel_format,  # type: ignore[arg-type]
        memory="metal",
        data=array,
    )

mlx_available() cached

True if MLX (Apple Metal) is usable in this process (cached). macOS + mlx importable.

Source code in src/pdum/rfb/metal.py 
30
31
32
33
34
35
36
37
38
39
40
@functools.lru_cache(maxsize=1)
def mlx_available() -> bool:
    """True if MLX (Apple Metal) is usable in this process (cached). macOS + ``mlx`` importable."""
    if sys.platform != "darwin" or importlib.util.find_spec("mlx") is None:
        return False
    try:
        import mlx.core as mx

        return "gpu" in str(mx.default_device()).lower()
    except Exception:
        return False

nvenc_cpu_available()

True if a usable NVENC H.264 encoder is present (cached).

Guards, in order: the OS must be one where NVENC exists (not macOS); PyAV must expose h264_nvenc; and the encoder must actually open and encode a frame on the GPU (which requires an NVENC-capable device and a working driver). The result is cached for the lifetime of the process.

Source code in src/pdum/rfb/encoders/nvenc_cpu.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
def nvenc_cpu_available() -> bool:
    """True if a usable NVENC H.264 encoder is present (cached).

    Guards, in order: the OS must be one where NVENC exists (not macOS); PyAV
    must expose ``h264_nvenc``; and the encoder must actually open and encode a
    frame on the GPU (which requires an NVENC-capable device and a working
    driver). The result is cached for the lifetime of the process.
    """
    global _nvenc_ok
    if _nvenc_ok is not None:
        return _nvenc_ok
    if not nvenc_codec_available():
        _nvenc_ok = False
        return _nvenc_ok
    _nvenc_ok = _probe_open()
    return _nvenc_ok

nvenc_gpu_pyav_available()

True if the zero-copy CUDA→NVENC path is usable (see :func:cuda_zerocopy_available).

Source code in src/pdum/rfb/encoders/nvenc_gpu_pyav.py 
36
37
38
def nvenc_gpu_pyav_available() -> bool:
    """True if the zero-copy CUDA→NVENC path is usable (see :func:`cuda_zerocopy_available`)."""
    return cuda_zerocopy_available()

pack_binary_message(header, payload)

Pack a header dict and payload bytes into a single binary message.

The header is encoded as compact UTF-8 JSON (no spaces) prefixed by its little-endian uint32 byte length.

Source code in src/pdum/rfb/protocol.py 
47
48
49
50
51
52
53
54
def pack_binary_message(header: dict, payload: bytes) -> bytes:
    """Pack a header dict and payload bytes into a single binary message.

    The header is encoded as compact UTF-8 JSON (no spaces) prefixed by its
    little-endian ``uint32`` byte length.
    """
    header_bytes = json.dumps(header, separators=(",", ":")).encode("utf-8")
    return struct.pack("<I", len(header_bytes)) + header_bytes + bytes(payload)

register_video_encoder(name, factory)

Register a video :class:EncoderBackend factory under name.

Source code in src/pdum/rfb/encoders/base.py 
24
25
26
def register_video_encoder(name: str, factory: EncoderFactory) -> None:
    """Register a video :class:`EncoderBackend` factory under ``name``."""
    _VIDEO_ENCODERS[name] = factory

select_transport(client_supported, *, has_h264, has_nvenc=False, prefer_video=True, image_mode='jpeg')

Choose the best backend given client capabilities and server encoders.

Policy (guide section 12): if the client supports WebCodecs/H.264, the server prefers video and at least one H.264 encoder is available, pick H.264 (NVENC is preferred over the CPU path when present). Otherwise fall back to the best mutually-supported image format. has_nvenc is accepted now so the NVENC backend can be slotted in later without touching callers.

Raises:

Type Description
UnsupportedClient

If no mutually-supported transport exists.
Source code in src/pdum/rfb/protocol.py 
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
def select_transport(
    client_supported: list[str],
    *,
    has_h264: bool,
    has_nvenc: bool = False,
    prefer_video: bool = True,
    image_mode: ImageMode = "jpeg",
) -> BackendSelection:
    """Choose the best backend given client capabilities and server encoders.

    Policy (guide section 12): if the client supports WebCodecs/H.264, the
    server prefers video and at least one H.264 encoder is available, pick
    H.264 (NVENC is preferred over the CPU path when present). Otherwise fall
    back to the best mutually-supported image format. ``has_nvenc`` is accepted
    now so the NVENC backend can be slotted in later without touching callers.

    Raises
    ------
    UnsupportedClient
        If no mutually-supported transport exists.
    """
    supported = set(client_supported)

    if prefer_video and CAP_H264_ANNEXB in supported and (has_nvenc or has_h264):
        return BackendSelection(transport="h264", codec=DEFAULT_H264_CODEC)

    # Prefer the caller's requested image mode if the client supports it,
    # then fall back to any mutually-supported image format.
    preferred_cap = _MIME_BY_MODE[image_mode]
    ordered_caps = [preferred_cap, CAP_PNG, CAP_JPEG, CAP_WEBP]
    for cap in ordered_caps:
        if cap in supported:
            mode = _MODE_BY_CAP[cap]
            return BackendSelection(transport="image", mime=cap, image_mode=mode)

    raise UnsupportedClient(f"no supported transport in client capabilities: {sorted(supported)}")

serve(width, height, *, host='127.0.0.1', port=8765, fps=30, bitrate=12000000, max_inflight=2, has_h264=None, has_nvenc=None, gpu=False, adaptive=False, still_after=None, stats_interval=None, authenticate=None, origins=None, record_events=False, event_log=None, event_queue_size=4096, own_frames=False, resize_policy='publisher', max_render_dimension=None, encode_pipeline_depth=0) async

Start the RFB WebSocket server in the background and return a :class:Display.

You own your loop: display = await serve(w, h, port=...) then call display.publish(frame) whenever you like, and await display.aclose() to shut down.

Parameters:

Name Type Description Default
width int

Initial framebuffer size (a connecting client is configured to the display's current size; publish a different shape to resize).

 required
height int

Initial framebuffer size (a connecting client is configured to the display's current size; publish a different shape to resize).

 required
has_h264 bool | None

None auto-detects; False forces the CPU/image fallback. has_nvenc selects the GPU encoder when an NVENC-capable device is present.

 None
has_nvenc bool | None

None auto-detects; False forces the CPU/image fallback. has_nvenc selects the GPU encoder when an NVENC-capable device is present.

 None
gpu bool

Opt in to GPU encode: the publisher pushes CUDA frames (CuPy/DLPack NV12 or rgb) and each viewer's H.264 encoder reads them directly, no host copy. Prefers the PyAV-free NVENC SDK backend (habemus-papadum-nvenc) when available, else the zero-copy CUDA→NVENC backend (PyAV >= 18). Validated at startup; raises if neither is usable. For the PyAV-18 path, call :func:pdum.rfb.gpu.enable_cuda_context_sharing before any CuPy use.

 False
still_after float | None

Opt in to "still after interaction settles": when no new frame is published for still_after seconds (e.g. 0.15), each viewer is sent a high-quality still of the resting frame — a lossless PNG on the image path, a clean IDR on the video path — so the settled image is crisp even though the live stream is lossy. None (default) disables it. See docs/still_after_settle.md.

 None
adaptive bool

Enable adaptive quality (bitrate → fps → in-flight, with recovery): the encoder is rebuilt as the controller reacts to the client's decode-queue depth and RTT. Pairs well with stats_interval so the browser can show it.

 False
stats_interval float | None

Opt in to a periodic server→client stats control message (seconds, e.g. 1.0) carrying authoritative server metrics — RTT, fps, bitrate, encode time, and the adaptive targets — so the browser can surface them in its Stats. None (default) sends none.

 None
authenticate Authenticator | None

Optional async hook (see :mod:pdum.rfb.auth); rejected connections are closed with code 4401 before any frame is sent.

 None
origins list[str | None] | None

Allowed Origin values (CSWSH defense) passed to websockets.

 None
own_frames bool

Opt in to server-owned frames: publish() copies each frame into a recycled server buffer so you may reuse/mutate your own buffer immediately after it returns (no reallocation, no "released" callback). Default False keeps the zero-copy borrow (publish a fresh buffer each call). cpu/cuda only; metal raises. See :meth:Display.publish.

 False
resize_policy str

Opt in to match-client resize: resize_policy="match_client" makes a viewer's set_viewport authoritative — the render stream follows the viewport via display.target_size (default "publisher" keeps you in charge of size). max_render_dimension caps the followed size. See :attr:Display.target_size.

 'publisher'
max_render_dimension str

Opt in to match-client resize: resize_policy="match_client" makes a viewer's set_viewport authoritative — the render stream follows the viewport via display.target_size (default "publisher" keeps you in charge of size). max_render_dimension caps the followed size. See :attr:Display.target_size.

 'publisher'
encode_pipeline_depth int

Encoder pipeline depth. 0 (default) is synchronous 1-in-1-out — lowest latency, optimal for the interactive latest-frame-wins model. > 0 opts into the token-based pipelined encode path on backends that implement it (NVENC, for throughput at high fps / many streams). On VideoToolbox it is correct but not faster (low-latency RC is synchronous). See docs/pipelined_encode.md.

 0
Notes

This hosts a single "default" stream. Reach the hub behind it via display.server to host several streams from the one port (display.server.add_stream("camera_b", 640, 480)), or start with :func:serve_server for a hub with no default stream. See docs/multiple_streams.md.

Source code in src/pdum/rfb/server.py 
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
async def serve(
    width: int,
    height: int,
    *,
    host: str = "127.0.0.1",
    port: int = 8765,
    fps: int = 30,
    bitrate: int = 12_000_000,
    max_inflight: int = 2,
    has_h264: bool | None = None,
    has_nvenc: bool | None = None,
    gpu: bool = False,
    adaptive: bool = False,
    still_after: float | None = None,
    stats_interval: float | None = None,
    authenticate: Authenticator | None = None,
    origins: list[str | None] | None = None,
    record_events: bool = False,
    event_log: str | Path | None = None,
    event_queue_size: int = 4096,
    own_frames: bool = False,
    resize_policy: str = "publisher",
    max_render_dimension: int | None = None,
    encode_pipeline_depth: int = 0,
) -> Display:
    """Start the RFB WebSocket server in the background and return a :class:`Display`.

    You own your loop: ``display = await serve(w, h, port=...)`` then call
    ``display.publish(frame)`` whenever you like, and ``await display.aclose()`` to
    shut down.

    Parameters
    ----------
    width, height:
        Initial framebuffer size (a connecting client is configured to the
        display's current size; publish a different shape to resize).
    has_h264, has_nvenc:
        ``None`` auto-detects; ``False`` forces the CPU/image fallback. ``has_nvenc``
        selects the GPU encoder when an NVENC-capable device is present.
    gpu:
        Opt in to **GPU encode**: the publisher pushes CUDA frames (CuPy/DLPack NV12
        or rgb) and each viewer's H.264 encoder reads them directly, no host copy.
        Prefers the **PyAV-free NVENC SDK** backend (``habemus-papadum-nvenc``) when
        available, else the **zero-copy CUDA→NVENC** backend (PyAV >= 18). Validated
        at startup; raises if neither is usable. For the PyAV-18 path, call
        :func:`pdum.rfb.gpu.enable_cuda_context_sharing` before any CuPy use.
    still_after:
        Opt in to **"still after interaction settles"**: when no new frame is
        published for ``still_after`` seconds (e.g. ``0.15``), each viewer is sent a
        high-quality still of the resting frame — a **lossless PNG** on the image
        path, a clean **IDR** on the video path — so the settled image is crisp even
        though the live stream is lossy. ``None`` (default) disables it. See
        ``docs/still_after_settle.md``.
    adaptive:
        Enable adaptive quality (bitrate → fps → in-flight, with recovery): the
        encoder is rebuilt as the controller reacts to the client's decode-queue
        depth and RTT. Pairs well with ``stats_interval`` so the browser can show it.
    stats_interval:
        Opt in to a periodic server→client ``stats`` control message (seconds, e.g.
        ``1.0``) carrying authoritative server metrics — RTT, fps, bitrate, encode
        time, and the adaptive targets — so the browser can surface them in its
        ``Stats``. ``None`` (default) sends none.
    authenticate:
        Optional async hook (see :mod:`pdum.rfb.auth`); rejected connections are
        closed with code ``4401`` before any frame is sent.
    origins:
        Allowed ``Origin`` values (CSWSH defense) passed to ``websockets``.
    own_frames:
        Opt in to **server-owned frames**: ``publish()`` copies each frame into a
        recycled server buffer so you may reuse/mutate your own buffer immediately after
        it returns (no reallocation, no "released" callback). Default ``False`` keeps the
        zero-copy borrow (publish a fresh buffer each call). ``cpu``/``cuda`` only;
        ``metal`` raises. See :meth:`Display.publish`.
    resize_policy, max_render_dimension:
        Opt in to **match-client resize**: ``resize_policy="match_client"`` makes a viewer's
        ``set_viewport`` authoritative — the render stream follows the viewport via
        ``display.target_size`` (default ``"publisher"`` keeps you in charge of size).
        ``max_render_dimension`` caps the followed size. See :attr:`Display.target_size`.
    encode_pipeline_depth:
        Encoder pipeline depth. ``0`` (default) is synchronous 1-in-1-out — lowest
        latency, optimal for the interactive latest-frame-wins model. ``> 0`` opts into
        the token-based **pipelined** encode path on backends that implement it (NVENC,
        for throughput at high fps / many streams). On VideoToolbox it is correct but not
        faster (low-latency RC is synchronous). See ``docs/pipelined_encode.md``.

    Notes
    -----
    This hosts a single ``"default"`` stream. Reach the hub behind it via
    ``display.server`` to host **several** streams from the one port
    (``display.server.add_stream("camera_b", 640, 480)``), or start with
    :func:`serve_server` for a hub with no default stream. See
    ``docs/multiple_streams.md``.
    """
    server = Server(host=host, port=port, origins=origins)
    display = server.add_stream(
        DEFAULT_STREAM,
        width,
        height,
        fps=fps,
        bitrate=bitrate,
        max_inflight=max_inflight,
        has_h264=has_h264,
        has_nvenc=has_nvenc,
        gpu=gpu,
        adaptive=adaptive,
        still_after=still_after,
        stats_interval=stats_interval,
        authenticate=authenticate,
        record_events=record_events,
        event_log=event_log,
        event_queue_size=event_queue_size,
        own_frames=own_frames,
        resize_policy=resize_policy,
        max_render_dimension=max_render_dimension,
        encode_pipeline_depth=encode_pipeline_depth,
    )
    await server.start()
    # The one-liner contract: closing the returned Display tears down the whole hub.
    display._server_cm = server
    return display

serve_server(*, host='127.0.0.1', port=8765, origins=None, streams=None) async

Start a multi-stream hub and return the :class:Server.

Unlike :func:serve (one default stream, returns its Display), this returns the hub itself with no default stream. Add streams with server.add_stream(name, w, h, **config) — each returns its own Display to publish into — and clients attach by URL path (ws://host/<name>). A client with no path is rejected until a "default" stream exists.

Parameters:

Name Type Description Default
host str

As for :func:serve — they configure the one shared listener.

 '127.0.0.1'
port str

As for :func:serve — they configure the one shared listener.

 '127.0.0.1'
origins str

As for :func:serve — they configure the one shared listener.

 '127.0.0.1'
streams list[dict[str, Any]] | None

Optional list of add_stream keyword dicts to register before the listener starts (atomic setup), e.g. [{"name": "rgb", "width": 1280, "height": 720, "gpu": True}]. You can also add streams afterwards.

 None

Examples:

>>> server = await serve_server(port=8765)
>>> cam = server.add_stream("camera", 1280, 720)
>>> depth = server.add_stream("depth", 640, 480, has_h264=False)
>>> cam.publish(render_camera()); depth.publish(render_depth())
>>> await server.aclose()
Source code in src/pdum/rfb/server.py 
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
async def serve_server(
    *,
    host: str = "127.0.0.1",
    port: int = 8765,
    origins: list[str | None] | None = None,
    streams: list[dict[str, Any]] | None = None,
) -> Server:
    """Start a **multi-stream hub** and return the :class:`Server`.

    Unlike :func:`serve` (one default stream, returns its ``Display``), this returns
    the hub itself with no default stream. Add streams with
    ``server.add_stream(name, w, h, **config)`` — each returns its own ``Display`` to
    publish into — and clients attach by URL path (``ws://host/<name>``). A client
    with no path is rejected until a ``"default"`` stream exists.

    Parameters
    ----------
    host, port, origins:
        As for :func:`serve` — they configure the one shared listener.
    streams:
        Optional list of ``add_stream`` keyword dicts to register **before** the
        listener starts (atomic setup), e.g.
        ``[{"name": "rgb", "width": 1280, "height": 720, "gpu": True}]``. You can
        also add streams afterwards.

    Examples
    --------
    >>> server = await serve_server(port=8765)
    >>> cam = server.add_stream("camera", 1280, 720)
    >>> depth = server.add_stream("depth", 640, 480, has_h264=False)
    >>> cam.publish(render_camera()); depth.publish(render_depth())
    >>> await server.aclose()
    """
    server = Server(host=host, port=port, origins=origins)
    for spec in streams or []:
        server.add_stream(**spec)
    await server.start()
    return server

unpack_binary_message(buf)

Inverse of :func:pack_binary_message.

Returns:

Type Description
tuple[dict, bytes]

The decoded JSON header and the raw payload bytes.
Source code in src/pdum/rfb/protocol.py 
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def unpack_binary_message(buf: bytes | bytearray | memoryview) -> tuple[dict, bytes]:
    """Inverse of :func:`pack_binary_message`.

    Returns
    -------
    tuple[dict, bytes]
        The decoded JSON header and the raw payload bytes.
    """
    mv = memoryview(buf)
    if len(mv) < 4:
        raise ValueError("buffer too small to contain a header length prefix")
    (n,) = struct.unpack("<I", mv[:4])
    if len(mv) < 4 + n:
        raise ValueError(f"buffer truncated: need {4 + n} bytes, have {len(mv)}")
    header = json.loads(bytes(mv[4 : 4 + n]).decode("utf-8"))
    payload = bytes(mv[4 + n :])
    return header, payload

adaptive

Adaptive quality control (guide section 10).

A small, pure controller that watches the client's decode-queue depth and round-trip latency and decides a new target quality. It is deliberately transport- and encoder-agnostic: it only emits a :class:QualityTarget; the session decides how to apply it (rebuild the H.264 encoder at the new bitrate, tighten the in-flight ceiling).

Four knobs, applied in order under congestion:

  • bitrate — the primary lever; reduce when congested, recover when healthy;
  • fps — once bitrate is at the floor and the client is still behind, lower the target frame rate (the encoder is rebuilt at the new rate, lightening both encode cost and bandwidth);
  • max in-flight — the latency lever; once bitrate and fps are at their floors, tighten the in-flight ceiling so latest-frame-wins drops more aggressively;
  • resolution scale — the deepest lever, engaged only under sustained pressure (bitrate, fps, and in-flight already floored). Unlike the others the session cannot apply it — the publisher owns the framebuffer size — so it surfaces the target :attr:QualityTarget.scale and the :class:~pdum.rfb.display.Display fans a :class:~pdum.rfb.types.DownscaleHint out through poll_events() for the render loop to honor (publish a smaller frame; the encoder rebuild + keyframe on resize is the well-trodden path). See :class:~pdum.rfb.types.DownscaleHint.

Recovery walks back up when healthy. A cooldown prevents thrashing (each bitrate / fps / resolution change costs a keyframe).

AdaptiveQualityController dataclass

Map observed metrics to a target quality with hysteresis + cooldown.

Source code in src/pdum/rfb/adaptive.py 
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
@dataclass
class AdaptiveQualityController:
    """Map observed metrics to a target quality with hysteresis + cooldown."""

    min_bitrate: int = 1_000_000
    max_bitrate: int = 12_000_000
    bitrate: int = 12_000_000

    min_inflight: int = 1
    max_inflight: int = 3
    inflight: int = 3

    min_fps: int = 10
    max_fps: int = 30
    fps: int = 30
    fps_step: int = 5

    # Resolution scale (deepest lever): 1.0 = full res, floored at min_scale. Absolute,
    # so the render loop applies it to its native size without compounding.
    min_scale: float = 0.5
    scale: float = 1.0
    scale_step: float = 0.25

    queue_high: int = 3  # decode_queue_size above this is "congested"
    rtt_high_ms: float = 150.0
    rtt_low_ms: float = 60.0

    cooldown_s: float = 1.0
    down_factor: float = 0.6
    up_factor: float = 1.25

    _last_change: float = -1e9

    def update(self, metrics: dict, *, now: float) -> QualityTarget | None:
        """Return a new target when a change is warranted, else ``None``."""
        if now - self._last_change < self.cooldown_s:
            return None

        queue = int(metrics.get("decode_queue_size", 0))
        rtt = float(metrics.get("rtt_ms", 0.0))
        congested = queue > self.queue_high or (rtt > 0 and rtt > self.rtt_high_ms)
        healthy = queue <= 1 and (rtt == 0 or rtt < self.rtt_low_ms)

        new_bitrate, new_inflight, new_fps = self.bitrate, self.inflight, self.fps
        new_scale = self.scale
        if congested:
            new_bitrate = max(self.min_bitrate, int(self.bitrate * self.down_factor))
            if new_bitrate == self.bitrate:  # bitrate floored; ease the frame rate
                new_fps = max(self.min_fps, self.fps - self.fps_step)
                if new_fps == self.fps:  # fps floored too; tighten latency
                    new_inflight = max(self.min_inflight, self.inflight - 1)
                    if new_inflight == self.inflight:  # everything floored; shrink resolution
                        new_scale = round(max(self.min_scale, self.scale - self.scale_step), 4)
        elif healthy:
            new_bitrate = min(self.max_bitrate, int(self.bitrate * self.up_factor))
            new_inflight = min(self.max_inflight, self.inflight + 1)
            new_fps = min(self.max_fps, self.fps + self.fps_step)
            new_scale = round(min(1.0, self.scale + self.scale_step), 4)
        else:
            return None

        if (new_bitrate, new_inflight, new_fps, new_scale) == (self.bitrate, self.inflight, self.fps, self.scale):
            return None

        self.bitrate, self.inflight, self.fps, self.scale = new_bitrate, new_inflight, new_fps, new_scale
        self._last_change = now
        return QualityTarget(bitrate=new_bitrate, max_inflight=new_inflight, fps=new_fps, scale=new_scale)

update(metrics, *, now)

Return a new target when a change is warranted, else None.

Source code in src/pdum/rfb/adaptive.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
def update(self, metrics: dict, *, now: float) -> QualityTarget | None:
    """Return a new target when a change is warranted, else ``None``."""
    if now - self._last_change < self.cooldown_s:
        return None

    queue = int(metrics.get("decode_queue_size", 0))
    rtt = float(metrics.get("rtt_ms", 0.0))
    congested = queue > self.queue_high or (rtt > 0 and rtt > self.rtt_high_ms)
    healthy = queue <= 1 and (rtt == 0 or rtt < self.rtt_low_ms)

    new_bitrate, new_inflight, new_fps = self.bitrate, self.inflight, self.fps
    new_scale = self.scale
    if congested:
        new_bitrate = max(self.min_bitrate, int(self.bitrate * self.down_factor))
        if new_bitrate == self.bitrate:  # bitrate floored; ease the frame rate
            new_fps = max(self.min_fps, self.fps - self.fps_step)
            if new_fps == self.fps:  # fps floored too; tighten latency
                new_inflight = max(self.min_inflight, self.inflight - 1)
                if new_inflight == self.inflight:  # everything floored; shrink resolution
                    new_scale = round(max(self.min_scale, self.scale - self.scale_step), 4)
    elif healthy:
        new_bitrate = min(self.max_bitrate, int(self.bitrate * self.up_factor))
        new_inflight = min(self.max_inflight, self.inflight + 1)
        new_fps = min(self.max_fps, self.fps + self.fps_step)
        new_scale = round(min(1.0, self.scale + self.scale_step), 4)
    else:
        return None

    if (new_bitrate, new_inflight, new_fps, new_scale) == (self.bitrate, self.inflight, self.fps, self.scale):
        return None

    self.bitrate, self.inflight, self.fps, self.scale = new_bitrate, new_inflight, new_fps, new_scale
    self._last_change = now
    return QualityTarget(bitrate=new_bitrate, max_inflight=new_inflight, fps=new_fps, scale=new_scale)

QualityTarget dataclass

A requested change to the session's encoding quality.

scale is the recommended render-resolution scale in (0, 1] (1.0 = full resolution). It is advisory to the publisher (surfaced as a :class:~pdum.rfb.types.DownscaleHint), not applied by the session like the other fields; defaults to 1.0 so existing callers are unaffected.

Source code in src/pdum/rfb/adaptive.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
@dataclass(slots=True)
class QualityTarget:
    """A requested change to the session's encoding quality.

    ``scale`` is the recommended render-resolution scale in ``(0, 1]`` (``1.0`` = full
    resolution). It is advisory to the *publisher* (surfaced as a
    :class:`~pdum.rfb.types.DownscaleHint`), not applied by the session like the other
    fields; defaults to ``1.0`` so existing callers are unaffected.
    """

    bitrate: int
    max_inflight: int
    fps: int
    scale: float = 1.0

asgi

ASGI / Starlette front-end: mount the framebuffer inside an existing app.

Opt-in (pip install habemus-papadum-rfb[asgi]) and additive: a second front-end over the exact same :class:~pdum.rfb.display.Display / :class:~pdum.rfb.session.RfbSession core as :func:pdum.rfb.serve. The standalone serve() path (with its zero-extra-deps websockets listener) is unchanged — this just lets you reach the same machinery through Starlette/FastAPI when you want same-origin hosting: shared TLS, routing, and the app's session/OAuth cookie (the auth hook receives AuthContext.cookies / .headers).

Because the ASGI server owns the event loop, the usage shape is: create your Display (or :class:~pdum.rfb.server.Server hub) at app startup, run your publish loop as a background task (e.g. from a lifespan handler), and mount one of the endpoints below. This module only adds the WebSocket endpoint that drives one RfbSession per connection.

Example (Starlette)::

import asyncio, contextlib, pdum.rfb as rfb
from pdum.rfb.asgi import rfb_endpoint
from starlette.applications import Starlette
from starlette.routing import WebSocketRoute

display = rfb.Display(1280, 720)

@contextlib.asynccontextmanager
async def lifespan(app):
    async def publish_loop():
        while True:
            display.publish(render())
            await asyncio.sleep(1 / 30)
    task = asyncio.create_task(publish_loop())
    yield
    task.cancel()

app = Starlette(lifespan=lifespan, routes=[
    WebSocketRoute("/rfb", rfb_endpoint(display, authenticate=my_cookie_auth)),
])

For several streams, mount :func:rfb_hub_endpoint on a path that captures a {stream} parameter.

rfb_endpoint(display, *, name=DEFAULT_STREAM, has_h264=None, has_nvenc=None, gpu=False, bitrate=12000000, fps=None, max_inflight=2, adaptive=False, still_after=None, stats_interval=None, authenticate=None)

Return a Starlette WebSocket endpoint that streams one :class:Display.

Mount it on any path::

app.add_websocket_route("/rfb", rfb_endpoint(display, authenticate=auth))

The keyword config mirrors :func:pdum.rfb.serve (encoder/transport selection, gpu, adaptive, still_after, per-endpoint authenticate). fps defaults to the display's. The authenticate hook receives an :class:~pdum.rfb.auth.AuthContext carrying the request cookies / headers so it can reuse the host app's same-origin session.

Source code in src/pdum/rfb/asgi.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
def rfb_endpoint(
    display: Display,
    *,
    name: str = DEFAULT_STREAM,
    has_h264: bool | None = None,
    has_nvenc: bool | None = None,
    gpu: bool = False,
    bitrate: int = 12_000_000,
    fps: int | None = None,
    max_inflight: int = 2,
    adaptive: bool = False,
    still_after: float | None = None,
    stats_interval: float | None = None,
    authenticate: Authenticator | None = None,
) -> Endpoint:
    """Return a Starlette WebSocket endpoint that streams one :class:`Display`.

    Mount it on any path::

        app.add_websocket_route("/rfb", rfb_endpoint(display, authenticate=auth))

    The keyword config mirrors :func:`pdum.rfb.serve` (encoder/transport selection,
    ``gpu``, ``adaptive``, ``still_after``, per-endpoint ``authenticate``). ``fps``
    defaults to the display's. The ``authenticate`` hook receives an
    :class:`~pdum.rfb.auth.AuthContext` carrying the request ``cookies`` / ``headers``
    so it can reuse the host app's same-origin session.
    """
    host = _stream_host(
        display,
        name,
        has_h264=has_h264,
        has_nvenc=has_nvenc,
        gpu=gpu,
        bitrate=bitrate,
        fps=fps,
        max_inflight=max_inflight,
        adaptive=adaptive,
        still_after=still_after,
        stats_interval=stats_interval,
        authenticate=authenticate,
    )

    async def endpoint(websocket: Any) -> None:
        await websocket.accept()
        await host._serve_connection(_AsgiConn(websocket))

    return endpoint

rfb_hub_endpoint(server, *, param='stream')

Return a Starlette endpoint routing to a :class:Server hub's streams.

Mount it on a path that captures the stream name as a path parameter::

app.add_websocket_route("/rfb/{stream}", rfb_hub_endpoint(server))

The connection is routed to server's stream of that name (an unknown stream closes with application code 4404); a request without the parameter uses the "default" stream. The hub and its per-stream config (including per-stream authenticate) are reused exactly as for the standalone listener.

Source code in src/pdum/rfb/asgi.py 
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
def rfb_hub_endpoint(server: Server, *, param: str = "stream") -> Endpoint:
    """Return a Starlette endpoint routing to a :class:`Server` hub's streams.

    Mount it on a path that captures the stream name as a path parameter::

        app.add_websocket_route("/rfb/{stream}", rfb_hub_endpoint(server))

    The connection is routed to ``server``'s stream of that name (an unknown stream
    closes with application code ``4404``); a request without the parameter uses the
    ``"default"`` stream. The hub and its per-stream config (including per-stream
    ``authenticate``) are reused exactly as for the standalone listener.
    """

    async def endpoint(websocket: Any) -> None:
        stream_name = websocket.path_params.get(param, DEFAULT_STREAM)
        host = server._streams.get(stream_name)
        await websocket.accept()
        if host is None:
            await websocket.close(4404, f"unknown stream {stream_name!r}")
            return
        await host._serve_connection(_AsgiConn(websocket))

    return endpoint

auth

Pluggable authentication seam (deliberately thin).

The library ships only the hook signature and the context/identity types — it never depends on a JWT/JWKS library and has no opinion on how you authenticate. You pass an authenticate callable to :func:pdum.rfb.serve; it is invoked once per connection and returns an application-defined principal (any object) to accept, or None to reject.

In v1 the credential arrives in the client's hello message (AuthContext.token) because a browser WebSocket cannot set request headers. The context also carries the handshake headers / path / query so a future same-site-cookie or ASGI transport can feed the same hook without an API change (at which point the hello token simply becomes optional).

Example — verify a Google OAuth ID token (your code; needs e.g. google-auth)::

from google.oauth2 import id_token
from google.auth.transport import requests as g_requests

ALLOWED = {"alice@example.com", "bob@example.com"}
_req = g_requests.Request()

async def authenticate(ctx):
    if not ctx.token:
        return None
    try:
        claims = id_token.verify_oauth2_token(ctx.token, _req, audience=CLIENT_ID)
    except ValueError:
        return None
    email = claims.get("email")
    return claims if email in ALLOWED else None

display = await rfb.serve(1280, 720, authenticate=authenticate)

AuthContext dataclass

Everything the auth hook may inspect about a connecting client.

Parameters:

Name Type Description Default
token str | None

The credential from the client's hello message (v1 transport).

 None
headers Mapping[str, str] | None

Handshake request headers (e.g. Cookie), when the transport exposes them. None for the plain hello-token path.

 None
cookies Mapping[str, str] | None

Parsed request cookies, when the transport exposes them (e.g. the ASGI adapter) — the natural home for a same-origin session/OAuth cookie.

 None
path str | None

Request path including query string, when available.

 None
query Mapping[str, str] | None

Parsed query parameters, when available.

 None
remote tuple[str, int] | None

(host, port) of the peer, when available.

 None
hello dict | None

The full decoded hello dict, for transports that carry auth in-band.

 None
stream str | None

Name of the stream (named :class:~pdum.rfb.display.Display) this client is connecting to, for per-stream authorization. "default" for the single-stream serve() path; the URL-path segment for a hub (ws://host/<stream>). See :func:pdum.rfb.serve_server.

 None
Source code in src/pdum/rfb/auth.py 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
@dataclass(slots=True)
class AuthContext:
    """Everything the auth hook may inspect about a connecting client.

    Parameters
    ----------
    token:
        The credential from the client's ``hello`` message (v1 transport).
    headers:
        Handshake request headers (e.g. ``Cookie``), when the transport exposes
        them. ``None`` for the plain ``hello``-token path.
    cookies:
        Parsed request cookies, when the transport exposes them (e.g. the ASGI
        adapter) — the natural home for a same-origin session/OAuth cookie.
    path:
        Request path including query string, when available.
    query:
        Parsed query parameters, when available.
    remote:
        ``(host, port)`` of the peer, when available.
    hello:
        The full decoded ``hello`` dict, for transports that carry auth in-band.
    stream:
        Name of the stream (named :class:`~pdum.rfb.display.Display`) this client is
        connecting to, for per-stream authorization. ``"default"`` for the
        single-stream ``serve()`` path; the URL-path segment for a hub
        (``ws://host/<stream>``). See :func:`pdum.rfb.serve_server`.
    """

    token: str | None = None
    headers: Mapping[str, str] | None = None
    cookies: Mapping[str, str] | None = None
    path: str | None = None
    query: Mapping[str, str] | None = None
    remote: tuple[str, int] | None = None
    hello: dict | None = None
    stream: str | None = None

benchmark

Offline encoder benchmark: image vs CPU H.264 vs GPU NVENC vs Apple VideoToolbox, fully headless.

Encodes a deterministic synthetic pattern and reports, per configuration:

  • encode time (mean and p95, milliseconds per frame),
  • payload size (mean bytes per frame),
  • the bitrate that size implies at a target frame rate,
  • quality as PSNR in dB, measured by decoding the output back (Pillow for images, PyAV for H.264) and comparing to the source — so quality is real, not assumed.

Run it directly::

uv run python -m pdum.rfb.benchmark --frames 120 --pattern gradient \
    --sizes 640x480,1280x720 --h264-bitrate 2M,8M

This has no network and no browser; it is the quickest way to characterize the software encoders.

benchmark_nvenc_gpu_pdum(*, bitrate=8000000, frames=60, width=640, height=480, fps=30, pattern='gradient')

NVIDIA Video Codec SDK encoder (habemus-papadum-nvenc / pdum.nvenc), no PyAV.

Frames are converted RGB→NV12 on the GPU and encoded straight from device memory (one intra-GPU copy into NVENC's input surface). Comparable to :func:benchmark_nvenc_gpu_pyav, but via NVIDIA's NvEncoderCuda instead of PyAV's h264_nvenc — and so works without PyAV>=18. See docs/proposals/completed/nvenc_sdk_evaluation.md.

Source code in src/pdum/rfb/benchmark.py 
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
def benchmark_nvenc_gpu_pdum(
    *,
    bitrate: int = 8_000_000,
    frames: int = 60,
    width: int = 640,
    height: int = 480,
    fps: int = 30,
    pattern: str = "gradient",
) -> BenchmarkResult:
    """NVIDIA Video Codec SDK encoder (habemus-papadum-nvenc / ``pdum.nvenc``), no PyAV.

    Frames are converted RGB→NV12 on the GPU and encoded straight from device
    memory (one intra-GPU copy into NVENC's input surface). Comparable to
    :func:`benchmark_nvenc_gpu_pyav`, but via NVIDIA's ``NvEncoderCuda`` instead of
    PyAV's ``h264_nvenc`` — and so works without PyAV>=18. See
    docs/proposals/completed/nvenc_sdk_evaluation.md.
    """
    from time import sleep

    import cupy as cp
    from pdum.nvenc import NvencEncoder

    from .gpu import rgb_to_nv12
    from .testing import decode_annexb

    src = _source_frames(pattern, frames, width, height)
    # Pre-upload RGB to the GPU (not timed); the timed region covers the on-GPU
    # RGB->NV12 conversion + the encode, matching benchmark_nvenc_gpu_pyav.
    rgb_frames = [cp.asarray(arr) for arr in src]
    cp.cuda.runtime.deviceSynchronize()

    def make_encoder():
        last: Exception | None = None
        for _ in range(4):  # consumer GPUs transiently EINVAL on session churn
            try:
                return NvencEncoder(
                    width, height, codec="h264", preset="p3", tuning="ll", fps=fps, gop=fps, bitrate=bitrate
                )
            except Exception as exc:  # pragma: no cover - hardware/driver dependent
                last = exc
                sleep(0.25)
        raise RuntimeError(f"NVENC SDK encoder failed to open after retries: {last}")

    enc = make_encoder()
    times: list[float] = []
    chunks: list[bytes] = []
    try:
        for seq, rgb in enumerate(rgb_frames):
            cp.cuda.runtime.deviceSynchronize()
            t0 = perf_counter()
            nv12 = rgb_to_nv12(rgb)
            payload = enc.encode(nv12, force_idr=(seq == 0))
            cp.cuda.runtime.deviceSynchronize()
            times.append((perf_counter() - t0) * 1000)
            if payload:
                chunks.append(payload)
        tail = enc.flush()
        if tail:
            chunks.append(tail)
    finally:
        enc.close()

    total_bytes = sum(len(c) for c in chunks)
    decoded = decode_annexb(b"".join(chunks))
    psnrs = [_psnr(src[i], f.to_ndarray(format="rgb24")) for i, f in enumerate(decoded[:frames])]
    bytes_per_frame = total_bytes / frames
    return BenchmarkResult(
        label=f"nvenc-gpu-pdum {bitrate // 1_000_000}M",
        encoder="nvenc-gpu-pdum",
        width=width,
        height=height,
        frames=frames,
        fps=fps,
        encode_ms_mean=float(np.mean(times)),
        encode_ms_p95=_p95(times),
        bytes_per_frame=bytes_per_frame,
        bitrate_at_fps_bps=bytes_per_frame * fps * 8,
        psnr_db=float(np.mean(psnrs)) if psnrs else float("nan"),
    )

benchmark_nvenc_gpu_pyav(*, bitrate=8000000, frames=60, width=640, height=480, fps=30, pattern='gradient')

Zero-copy CUDA→NVENC: frames pre-uploaded to the GPU, encoded with no host copy.

Mirrors :func:benchmark_nvenc but the per-frame timing covers the on-GPU RGB→NV12 conversion + the zero-copy encode (the realistic "everything on GPU" cost), so the row is directly comparable to the host nvenc row (whose cost includes the CPU rgb→yuv reformat + the PCIe upload).

Source code in src/pdum/rfb/benchmark.py 
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
def benchmark_nvenc_gpu_pyav(
    *,
    bitrate: int = 8_000_000,
    frames: int = 60,
    width: int = 640,
    height: int = 480,
    fps: int = 30,
    pattern: str = "gradient",
) -> BenchmarkResult:
    """Zero-copy CUDA→NVENC: frames pre-uploaded to the GPU, encoded with no host copy.

    Mirrors :func:`benchmark_nvenc` but the per-frame timing covers the on-GPU
    RGB→NV12 conversion + the zero-copy encode (the realistic "everything on GPU"
    cost), so the row is directly comparable to the host ``nvenc`` row (whose cost
    includes the CPU ``rgb→yuv`` reformat + the PCIe upload).
    """
    from time import sleep

    import cupy as cp

    from .encoders.nvenc_gpu_pyav import NvencGpuPyavEncoder
    from .gpu import cuda_frame, enable_cuda_context_sharing
    from .testing import decode_annexb

    enable_cuda_context_sharing()
    src = _source_frames(pattern, frames, width, height)
    device_frames = [cuda_frame(cp.asarray(arr), seq=seq) for seq, arr in enumerate(src)]
    cp.cuda.runtime.deviceSynchronize()

    def make_encoder():
        last: Exception | None = None
        for _ in range(4):  # consumer GPUs transiently EINVAL on session churn
            try:
                return NvencGpuPyavEncoder(width=width, height=height, fps=fps, bitrate=bitrate)
            except ValueError:
                raise
            except Exception as exc:  # pragma: no cover - hardware/driver dependent
                last = exc
                sleep(0.25)
        raise RuntimeError(f"CUDA NVENC encoder failed to open after retries: {last}")

    enc = make_encoder()
    times: list[float] = []
    total_bytes = 0
    chunks: list[bytes] = []
    for seq, frame in enumerate(device_frames):
        cp.cuda.runtime.deviceSynchronize()
        t0 = perf_counter()
        payloads = enc.encode(frame, force_keyframe=(seq == 0))
        cp.cuda.runtime.deviceSynchronize()
        times.append((perf_counter() - t0) * 1000)
        for p in payloads:
            total_bytes += len(p.payload)
            chunks.append(p.payload)
    for p in enc.flush():
        total_bytes += len(p.payload)
        chunks.append(p.payload)
    enc.close()

    decoded = decode_annexb(b"".join(chunks))
    psnrs = [_psnr(src[i], f.to_ndarray(format="rgb24")) for i, f in enumerate(decoded[:frames])]
    bytes_per_frame = total_bytes / frames
    return BenchmarkResult(
        label=f"nvenc-gpu-pyav {bitrate // 1_000_000}M",
        encoder="nvenc-gpu-pyav",
        width=width,
        height=height,
        frames=frames,
        fps=fps,
        encode_ms_mean=float(np.mean(times)),
        encode_ms_p95=_p95(times),
        bytes_per_frame=bytes_per_frame,
        bitrate_at_fps_bps=bytes_per_frame * fps * 8,
        psnr_db=float(np.mean(psnrs)) if psnrs else float("nan"),
    )

benchmark_vtenc(*, bitrate=8000000, frames=60, width=640, height=480, fps=30, pattern='gradient', use_mlx=None)

Apple VideoToolbox H.264 (macOS) — the counterpart of the NVENC GPU rows.

When MLX is available (use_mlx None auto-detects), the RGB→NV12 conversion runs on the GPU — the serve(gpu=True) path — and the timed region covers that GPU convert and the VideoToolbox encode, so the vtenc-gpu row is directly comparable to the NVENC GPU rows. Without MLX the conversion is on the CPU (vtenc-cpu). Decoded back with PyAV for real PSNR. See docs/metal_videotoolbox.md.

Source code in src/pdum/rfb/benchmark.py 
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
def benchmark_vtenc(
    *,
    bitrate: int = 8_000_000,
    frames: int = 60,
    width: int = 640,
    height: int = 480,
    fps: int = 30,
    pattern: str = "gradient",
    use_mlx: bool | None = None,
) -> BenchmarkResult:
    """Apple VideoToolbox H.264 (macOS) — the counterpart of the NVENC GPU rows.

    When MLX is available (``use_mlx`` ``None`` auto-detects), the RGB→NV12 conversion
    runs **on the GPU** — the ``serve(gpu=True)`` path — and the timed region covers that
    GPU convert **and** the VideoToolbox encode, so the ``vtenc-gpu`` row is directly
    comparable to the NVENC GPU rows. Without MLX the conversion is on the CPU
    (``vtenc-cpu``). Decoded back with PyAV for real PSNR. See docs/metal_videotoolbox.md.
    """
    from .encoders.vtenc import VideoToolboxEncoder
    from .metal import mlx_available
    from .testing import decode_annexb

    if use_mlx is None:
        use_mlx = mlx_available()

    src = _source_frames(pattern, frames, width, height)

    if use_mlx:
        import mlx.core as mx

        from .metal import metal_frame

        # Pre-upload RGB to the GPU as MLX arrays (not timed); the timed region below
        # covers the on-GPU RGB→NV12 conversion + the encode, matching the NVENC GPU rows.
        inputs: list[RawFrame] = [
            metal_frame(mx.array(arr), pixel_format="rgb24", seq=seq) for seq, arr in enumerate(src)
        ]
        mx.eval(*[f.data for f in inputs])
    else:
        inputs = [RawFrame(seq, width, height, seq * 1000, "rgb24", "cpu", arr) for seq, arr in enumerate(src)]

    enc = VideoToolboxEncoder(width=width, height=height, fps=fps, bitrate=bitrate)
    times: list[float] = []
    total_bytes = 0
    chunks: list[bytes] = []
    try:
        for seq, frame in enumerate(inputs):
            t0 = perf_counter()
            payloads = enc.encode(frame, force_keyframe=(seq == 0))
            times.append((perf_counter() - t0) * 1000)
            for p in payloads:
                total_bytes += len(p.payload)
                chunks.append(p.payload)
        for p in enc.flush():
            total_bytes += len(p.payload)
            chunks.append(p.payload)
    finally:
        enc.close()

    decoded = decode_annexb(b"".join(chunks))
    psnrs = [_psnr(src[i], f.to_ndarray(format="rgb24")) for i, f in enumerate(decoded[:frames])]
    bytes_per_frame = total_bytes / frames
    return BenchmarkResult(
        label=f"vtenc-{'gpu' if use_mlx else 'cpu'} {bitrate // 1_000_000}M",
        encoder="vtenc",
        width=width,
        height=height,
        frames=frames,
        fps=fps,
        encode_ms_mean=float(np.mean(times)),
        encode_ms_p95=_p95(times),
        bytes_per_frame=bytes_per_frame,
        bitrate_at_fps_bps=bytes_per_frame * fps * 8,
        psnr_db=float(np.mean(psnrs)) if psnrs else float("nan"),
    )

cli

pdum-rfb command-line entry point.

Commands (all optional — install with pip install habemus-papadum-rfb[cli] or [demo]):

  • pdum-rfb doctor — probe this box and show, as a table, which encode paths work (image, CPU H.264, host NVENC, zero-copy CUDA→NVENC, NVENC SDK, and — on macOS/Apple Silicon — Apple VideoToolbox + MLX) and which one to prefer.
  • pdum-rfb benchmark — measure per-frame encode latency / size / PSNR for every available path (a Rich-rendered wrapper over :mod:pdum.rfb.benchmark).
  • pdum-rfb demo — the self-contained web app ([demo] extra).
  • pdum-rfb jupyter-demo / marimo-demo — copy the bundled "renderer + viewer in one kernel" notebook to a working dir and launch JupyterLab / marimo on it ([demo] extra).

The module imports cleanly without Typer/Rich; the console-script entry point then prints an install hint instead of crashing.

benchmark(sizes=typer.Option('1280x720,1920x1080', help='comma-separated WxH'), frames=typer.Option(120, help='frames per configuration'), fps=typer.Option(30, help='target frame rate'), bitrate=typer.Option('8M', help='H.264/NVENC target bitrate, e.g. 8M'), pattern=typer.Option('gradient'), jpeg_quality=typer.Option(80, help='JPEG quality for the image row'), image=typer.Option(True, help='include the image (JPEG) path'))

Benchmark every available encode path on this box (latency, size, PSNR).

Source code in src/pdum/rfb/cli.py 
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
@app.command()
def benchmark(
    sizes: str = typer.Option("1280x720,1920x1080", help="comma-separated WxH"),
    frames: int = typer.Option(120, help="frames per configuration"),
    fps: int = typer.Option(30, help="target frame rate"),
    bitrate: str = typer.Option("8M", help="H.264/NVENC target bitrate, e.g. 8M"),
    pattern: str = typer.Option("gradient"),
    jpeg_quality: int = typer.Option(80, help="JPEG quality for the image row"),
    image: bool = typer.Option(True, help="include the image (JPEG) path"),
) -> None:
    """Benchmark every available encode path on this box (latency, size, PSNR)."""
    from . import benchmark as bench

    console = Console()
    br = bench._parse_bitrate(bitrate)

    # detect what's available
    from .encoders.h264_cpu import h264_cpu_available
    from .encoders.nvenc_cpu import NVENC_MIN_WIDTH, nvenc_cpu_available
    from .encoders.vtenc import VTENC_MIN_WIDTH

    have_h264 = h264_cpu_available()
    have_nvenc = nvenc_cpu_available()
    have_gpu = bench._cuda_zerocopy_available()
    have_sdk = bench._nvenc_gpu_pdum_available()
    have_vtenc = bench._vtenc_available()

    results = []
    with console.status("[bold]benchmarking…"):
        for size in sizes.split(","):
            w, h = bench._parse_size(size)
            if image:
                results.append(
                    bench.benchmark_image(
                        mode="jpeg",
                        quality=jpeg_quality,
                        frames=frames,
                        width=w,
                        height=h,
                        fps=fps,
                        pattern=pattern,
                    )
                )
            if have_h264:
                results.append(
                    bench.benchmark_h264(bitrate=br, frames=frames, width=w, height=h, fps=fps, pattern=pattern)
                )
            if have_nvenc and w >= NVENC_MIN_WIDTH:
                results.append(
                    bench.benchmark_nvenc(bitrate=br, frames=frames, width=w, height=h, fps=fps, pattern=pattern)
                )
            if have_gpu and w >= NVENC_MIN_WIDTH:
                results.append(
                    bench.benchmark_nvenc_gpu_pyav(
                        bitrate=br, frames=frames, width=w, height=h, fps=fps, pattern=pattern
                    )
                )
            if have_sdk and w >= NVENC_MIN_WIDTH:
                results.append(
                    bench.benchmark_nvenc_gpu_pdum(
                        bitrate=br, frames=frames, width=w, height=h, fps=fps, pattern=pattern
                    )
                )
            if have_vtenc and w >= VTENC_MIN_WIDTH:
                results.append(
                    bench.benchmark_vtenc(bitrate=br, frames=frames, width=w, height=h, fps=fps, pattern=pattern)
                )

    table = Table(title=f"pdum.rfb encoders — {pattern}, {frames} frames @ {fps}fps", title_style="bold")
    for col in ("config", "size", "enc ms", "p95 ms", "KB/frame", "Mbps@fps", "PSNR dB"):
        table.add_column(col, justify="right" if col != "config" else "left")
    for r in results:
        psnr = "inf" if r.psnr_db == float("inf") else f"{r.psnr_db:.2f}"
        style = "green" if r.encoder.startswith(("nvenc", "vtenc")) else None
        table.add_row(
            r.label,
            f"{r.width}x{r.height}",
            f"{r.encode_ms_mean:.2f}",
            f"{r.encode_ms_p95:.2f}",
            f"{r.bytes_per_frame / 1024:.1f}",
            f"{r.bitrate_at_fps_bps / 1e6:.2f}",
            psnr,
            style=style,
        )
    console.print(table)
    skipped = [
        n
        for n, ok in (
            ("h264-cpu", have_h264),
            ("nvenc-cpu", have_nvenc),
            ("nvenc-gpu-pyav", have_gpu),
            ("nvenc-gpu-pdum", have_sdk),
            ("vtenc", have_vtenc),
        )
        if not ok
    ]
    if skipped:
        console.print(f"[dim]skipped (unavailable): {', '.join(skipped)} — run `pdum-rfb doctor`[/]")

demo(width=typer.Option(1280, help='framebuffer width (even)'), height=typer.Option(720, help='framebuffer height (even)'), port=typer.Option(0, help='HTTP/WebSocket server port (0 = pick a free one)'), fps=typer.Option(30, help='publish frame rate'), bitrate=typer.Option('8M', help='initial H.264/NVENC bitrate, e.g. 8M'), host=typer.Option('127.0.0.1', help='bind host (default localhost-only)'), verbose=typer.Option(False, '--verbose', '-v', help='DEBUG-level server logging'), open_browser=typer.Option(True, '--open/--no-open', help='open the browser at the demo URL'), dev=typer.Option(False, '--dev', help='live-reload agentic mode: Vite HMR (TS) + uvicorn reload (Python); needs repo + Node'), smoke=typer.Option(False, help='headless self-test: every backend + REST control, no browser'))

Interactive web demo: one process serves the app + controls; drive it in the browser.

The control plane (scene / backend / quality / parameters) lives in the browser and rides REST; Python only serves the app and logs. Ships prebuilt — run it with uvx::

uvx --from 'habemus-papadum-rfb[demo]' pdum-rfb demo

For pair-debugging with an agent, --dev runs the SPA under Vite (instant TS HMR) and the API under uvicorn reload=True (Python auto-restart) so edits to either side are picked up live; the browser opens on the Vite URL and proxies REST + WS to Python.

Source code in src/pdum/rfb/cli.py 
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
@app.command()
def demo(
    width: int = typer.Option(1280, help="framebuffer width (even)"),
    height: int = typer.Option(720, help="framebuffer height (even)"),
    port: int = typer.Option(0, help="HTTP/WebSocket server port (0 = pick a free one)"),
    fps: int = typer.Option(30, help="publish frame rate"),
    bitrate: str = typer.Option("8M", help="initial H.264/NVENC bitrate, e.g. 8M"),
    host: str = typer.Option("127.0.0.1", help="bind host (default localhost-only)"),
    verbose: bool = typer.Option(False, "--verbose", "-v", help="DEBUG-level server logging"),
    open_browser: bool = typer.Option(True, "--open/--no-open", help="open the browser at the demo URL"),
    dev: bool = typer.Option(
        False, "--dev", help="live-reload agentic mode: Vite HMR (TS) + uvicorn reload (Python); needs repo + Node"
    ),
    smoke: bool = typer.Option(False, help="headless self-test: every backend + REST control, no browser"),
) -> None:
    """Interactive web demo: one process serves the app + controls; drive it in the browser.

    The control plane (scene / backend / quality / parameters) lives in the browser and
    rides REST; Python only serves the app and logs. Ships prebuilt — run it with uvx::

        uvx --from 'habemus-papadum-rfb[demo]' pdum-rfb demo

    For pair-debugging with an agent, ``--dev`` runs the SPA under Vite (instant TS HMR) and
    the API under uvicorn ``reload=True`` (Python auto-restart) so edits to either side are
    picked up live; the browser opens on the Vite URL and proxies REST + WS to Python.
    """
    from . import demo_server

    w = width - (width % 2)
    h = height - (height % 2)
    if smoke:
        demo_server.smoke(width=w, height=h, fps=fps)
        return
    try:
        import uvicorn  # noqa: F401
    except ModuleNotFoundError:
        sys.stderr.write(
            "The demo needs Starlette + uvicorn.\n"
            "  uvx --from 'habemus-papadum-rfb[demo]' pdum-rfb demo\n"
            "  (or: pip install 'habemus-papadum-rfb[demo]')\n"
        )
        raise SystemExit(1) from None
    try:
        demo_server.run_demo(
            width=w,
            height=h,
            host=host,
            port=port,
            fps=fps,
            bitrate=bitrate,
            verbose=verbose,
            open_browser=open_browser,
            dev=dev,
        )
    except KeyboardInterrupt:
        pass

doctor()

Probe this box and report which encode paths work.

Source code in src/pdum/rfb/cli.py 
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
@app.command()
def doctor() -> None:
    """Probe this box and report which encode paths work."""
    console = Console()
    probes, rec = _probe_all()
    table = Table(title="pdum.rfb — encode path doctor", title_style="bold")
    table.add_column("Component", style="cyan", no_wrap=True)
    table.add_column("Status", no_wrap=True)
    table.add_column("Detail")
    for p in probes:
        table.add_row(p.component, _STATUS_MARKUP.get(p.status, p.status), p.detail)
    console.print(table)
    console.print(Panel(f"[bold]Recommended:[/] {rec}", border_style="green", expand=False))
    # "✓ ok" means importable *in this environment*; a "– n/a"/"△" with a platform note
    # means the path is possible on this box once you install it. Run from a fresh env
    # with the cross-platform encoders present via the compound [doctor] extra:
    console.print(
        "[dim]✓ = available in this environment · notes show what this platform could run "
        "if installed.\n"
        "Fresh probe:  uvx --from 'habemus-papadum-rfb[doctor]' pdum-rfb doctor[/]"
    )

jupyter_demo(dir=typer.Option('', '--dir', help='working dir for the notebook copy (default: a temp dir)'), open_browser=typer.Option(True, '--open/--no-open', help='open the browser (JupyterLab default)'))

Launch JupyterLab on the bundled 'renderer + viewer in one kernel' demo notebook.

Copies jupyter-demo.ipynb to a working dir and starts JupyterLab there::

uvx --from 'habemus-papadum-rfb[demo]' pdum-rfb jupyter-demo

The viewer connects to a WebSocket opened by the kernel — fine on a local notebook; in a hosted environment that port must be browser-reachable (see docs/notebook.md).

Source code in src/pdum/rfb/cli.py 
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
@app.command("jupyter-demo")
def jupyter_demo(
    dir: str = typer.Option("", "--dir", help="working dir for the notebook copy (default: a temp dir)"),
    open_browser: bool = typer.Option(True, "--open/--no-open", help="open the browser (JupyterLab default)"),
) -> None:
    """Launch JupyterLab on the bundled 'renderer + viewer in one kernel' demo notebook.

    Copies ``jupyter-demo.ipynb`` to a working dir and starts JupyterLab there::

        uvx --from 'habemus-papadum-rfb[demo]' pdum-rfb jupyter-demo

    The viewer connects to a WebSocket opened by the kernel — fine on a local notebook; in a
    hosted environment that port must be browser-reachable (see docs/notebook.md).
    """
    _run_notebook_demo("jupyter", dir, open_browser)

marimo_demo(dir=typer.Option('', '--dir', help='working dir for the notebook copy (default: a temp dir)'), open_browser=typer.Option(True, '--open/--no-open', help='open the browser (marimo default)'))

Launch marimo on the bundled 'renderer + viewer in one process' demo notebook.

Same wiring as the Jupyter demo, in marimo's reactive editor::

uvx --from 'habemus-papadum-rfb[demo]' pdum-rfb marimo-demo
Source code in src/pdum/rfb/cli.py 
477
478
479
480
481
482
483
484
485
486
487
488
@app.command("marimo-demo")
def marimo_demo(
    dir: str = typer.Option("", "--dir", help="working dir for the notebook copy (default: a temp dir)"),
    open_browser: bool = typer.Option(True, "--open/--no-open", help="open the browser (marimo default)"),
) -> None:
    """Launch marimo on the bundled 'renderer + viewer in one process' demo notebook.

    Same wiring as the Jupyter demo, in marimo's reactive editor::

        uvx --from 'habemus-papadum-rfb[demo]' pdum-rfb marimo-demo
    """
    _run_notebook_demo("marimo", dir, open_browser)

demo_server

The pdum-rfb demo web app: a Starlette control plane over a framebuffer hub.

One pdum-rfb demo process serves a self-contained web app — the prebuilt SPA (built under static/demo/ via pnpm -C widgets build:demo; a build artifact, not committed — the release CI builds it into the wheel), a small REST control plane, and the framebuffer WebSocket(s) — all on one origin. There is no Node, no Vite, and no terminal UI: the browser holds both the remote-framebuffer viewer and the controls (scene / encode backend / quality / the richer parameters), and drives the server with REST calls. The Python side only serves the app and logs lifecycle to stdout.

Architecture
  • A :class:~pdum.rfb.server.Server hub owns the named streams, but its websockets listener is never started — every connection is driven through Starlette via :func:~pdum.rfb.asgi.rfb_hub_endpoint, so REST + static + WS share one uvicorn origin.
  • A :class:DemoStreamManager owns, per stream, the :class:_DemoState (which scene is live) and the publish task (:func:_render_loop).
  • Streams model the two multi-client modes. The shared "default" stream is coupled — many viewers see the same frames and any client's control affects them all (fan-out). A client can also mint a private stream (its own scene / backend / and the structural parameters chosen at birth), so two tabs can compare backends side by side. Private streams are reaped a short grace period after their last viewer leaves and are capped to bound resources.
Live vs structural parameters

scene / backend / bitrate / fps / resolution / color are cheap to change on a running stream (the existing live-switch / resize / per-frame-tag paths), so they are editable anytime. The structural parameters (adaptive, still_after, stats_interval, encode_pipeline_depth, resize_policy) are fixed at add_stream time — you explore them by creating a private stream.

A headless :func:smoke drives the real ASGI app in-process (Starlette TestClient): capabilities, every backend switched over REST on one socket, a multi-viewer fan-out check, and a private-stream create → connect → destroy cycle. It is the CI-grade proof.

DemoStreamManager

Owns the demo's streams: their scenes, publish tasks, and private-stream lifecycle.

Source code in src/pdum/rfb/demo_server.py 
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
class DemoStreamManager:
    """Owns the demo's streams: their scenes, publish tasks, and private-stream lifecycle."""

    def __init__(
        self,
        server: Server,
        *,
        default_width: int,
        default_height: int,
        fps: int = 30,
        bitrate: int = 8_000_000,
        stats_interval: float | None = 1.0,
        private_cap: int = 8,
        reap_grace: float = 10.0,
    ) -> None:
        self.server = server
        self.default_size = (_even(default_width), _even(default_height))
        self.fps = fps
        self.bitrate = bitrate
        self.stats_interval = stats_interval
        self.private_cap = private_cap
        self.reap_grace = reap_grace
        self.states: dict[str, _DemoState] = {}
        self.tasks: dict[str, asyncio.Task] = {}
        self._empty_since: dict[str, float] = {}
        self._private_seq = 0
        self._reaper: asyncio.Task | None = None

    def _first_scene(self) -> str:
        demos = available_demos()
        return demos[0].key if demos else DEMOS[0].key

    # --- create / destroy --------------------------------------------------

    def create(
        self,
        name: str,
        *,
        width: int,
        height: int,
        adaptive: bool = False,
        still_after: float | None = None,
        stats_interval: float | None = None,
        encode_pipeline_depth: int = 0,
        resize_policy: str = "publisher",
        max_render_dimension: int | None = None,
        private: bool = False,
    ) -> Any:
        """Register a stream on the hub, seed its scene, and start its publish task."""
        width, height = _even(width), _even(height)
        display = self.server.add_stream(
            name,
            width,
            height,
            fps=self.fps,
            bitrate=self.bitrate,
            adaptive=adaptive,
            still_after=still_after,
            stats_interval=self.stats_interval if stats_interval is None else stats_interval,
            encode_pipeline_depth=encode_pipeline_depth,
            resize_policy=resize_policy,
            max_render_dimension=max_render_dimension,
        )
        host = self.server._streams[name]
        state = _DemoState(self._first_scene(), width, height)
        self.states[name] = state
        self.tasks[name] = asyncio.create_task(_render_loop(display, state, host))
        log.info(
            "stream %r created: %dx%d scene=%s adaptive=%s still_after=%s resize=%s private=%s",
            name,
            width,
            height,
            state.active_key,
            adaptive,
            still_after,
            resize_policy,
            private,
        )
        return display

    def create_private(self, body: dict[str, Any]) -> str:
        """Mint a new private stream from a create-request body; returns its name.

        Raises :class:`RuntimeError` when the private-stream cap is reached.
        """
        private = [n for n in self.states if n != DEFAULT_STREAM]
        if len(private) >= self.private_cap:
            raise RuntimeError(f"private stream cap reached ({self.private_cap})")
        self._private_seq += 1
        name = f"s{self._private_seq}"
        dw, dh = self.default_size
        self.create(
            name,
            width=int(body.get("width", dw)),
            height=int(body.get("height", dh)),
            adaptive=bool(body.get("adaptive", False)),
            still_after=_opt_float(body.get("still_after")),
            stats_interval=_opt_float(body.get("stats_interval"), default=self.stats_interval),
            encode_pipeline_depth=int(body.get("encode_pipeline_depth", 0)),
            resize_policy=str(body.get("resize_policy", "publisher")),
            max_render_dimension=_opt_int(body.get("max_render_dimension")),
            private=True,
        )
        return name

    async def destroy(self, name: str) -> None:
        """Cancel a private stream's publish task and remove it from the hub."""
        if name == DEFAULT_STREAM:
            raise ValueError("cannot destroy the default stream")
        task = self.tasks.pop(name, None)
        if task is not None:
            task.cancel()
            with contextlib.suppress(asyncio.CancelledError):
                await task
        self.states.pop(name, None)
        self._empty_since.pop(name, None)
        self.server.remove_stream(name)
        log.info("stream %r destroyed", name)

    # --- lifecycle ---------------------------------------------------------

    async def start(self) -> None:
        dw, dh = self.default_size
        self.create(DEFAULT_STREAM, width=dw, height=dh, stats_interval=self.stats_interval, private=False)
        self._reaper = asyncio.create_task(self._reap_loop())

    async def aclose(self) -> None:
        if self._reaper is not None:
            self._reaper.cancel()
            with contextlib.suppress(asyncio.CancelledError):
                await self._reaper
            self._reaper = None
        for name in list(self.tasks):
            task = self.tasks.pop(name)
            task.cancel()
            with contextlib.suppress(asyncio.CancelledError):
                await task

    async def _reap_loop(self) -> None:
        """Destroy private streams that have had no viewers for ``reap_grace`` seconds."""
        while True:
            await asyncio.sleep(2.0)
            now = time.monotonic()
            for name in [n for n in self.states if n != DEFAULT_STREAM]:
                host = self.server._streams.get(name)
                if host is None:
                    continue
                if host.display.client_count == 0:
                    self._empty_since.setdefault(name, now)
                    if now - self._empty_since[name] >= self.reap_grace:
                        with contextlib.suppress(Exception):
                            await self.destroy(name)
                else:
                    self._empty_since.pop(name, None)

    # --- state reporting ---------------------------------------------------

    def stream_state(self, name: str) -> dict[str, Any]:
        host = self.server._streams[name]
        d = host.display
        st = self.states[name]
        return {
            "name": name,
            "ws": f"/rfb/{name}",
            "private": name != DEFAULT_STREAM,
            "clients": d.client_count,
            "scene": st.active_key,
            "backend": _current_backend(host),
            "bitrate": host.bitrate,
            "bitrate_label": _fmt_bitrate(host.bitrate),
            "fps": host.fps,
            # The DESIRED size (what the Size control set) under publisher policy — reading the
            # Display's actual d.width/d.height would lag one render tick after set_params, so the
            # panel dropdown would snap back to the old value. Under match_client the viewer truly
            # drives the size, so report the Display's live dimensions there.
            "width": d.width if d.resize_policy == "match_client" else st.width,
            "height": d.height if d.resize_policy == "match_client" else st.height,
            "color": st.color or "srgb",
            "adaptive": host.adaptive,
            "still_after": host.still_after,
            "stats_interval": host.stats_interval,
            "encode_pipeline_depth": host.encode_pipeline_depth,
            "resize_policy": d.resize_policy,
            "max_render_dimension": d.max_render_dimension,
            "last_error": st.last_error,
        }

    def state(self) -> dict[str, Any]:
        return {"streams": [self.stream_state(n) for n in self.states]}

create(name, *, width, height, adaptive=False, still_after=None, stats_interval=None, encode_pipeline_depth=0, resize_policy='publisher', max_render_dimension=None, private=False)

Register a stream on the hub, seed its scene, and start its publish task.

Source code in src/pdum/rfb/demo_server.py 
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
def create(
    self,
    name: str,
    *,
    width: int,
    height: int,
    adaptive: bool = False,
    still_after: float | None = None,
    stats_interval: float | None = None,
    encode_pipeline_depth: int = 0,
    resize_policy: str = "publisher",
    max_render_dimension: int | None = None,
    private: bool = False,
) -> Any:
    """Register a stream on the hub, seed its scene, and start its publish task."""
    width, height = _even(width), _even(height)
    display = self.server.add_stream(
        name,
        width,
        height,
        fps=self.fps,
        bitrate=self.bitrate,
        adaptive=adaptive,
        still_after=still_after,
        stats_interval=self.stats_interval if stats_interval is None else stats_interval,
        encode_pipeline_depth=encode_pipeline_depth,
        resize_policy=resize_policy,
        max_render_dimension=max_render_dimension,
    )
    host = self.server._streams[name]
    state = _DemoState(self._first_scene(), width, height)
    self.states[name] = state
    self.tasks[name] = asyncio.create_task(_render_loop(display, state, host))
    log.info(
        "stream %r created: %dx%d scene=%s adaptive=%s still_after=%s resize=%s private=%s",
        name,
        width,
        height,
        state.active_key,
        adaptive,
        still_after,
        resize_policy,
        private,
    )
    return display

create_private(body)

Mint a new private stream from a create-request body; returns its name.

Raises :class:RuntimeError when the private-stream cap is reached.

Source code in src/pdum/rfb/demo_server.py 
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
def create_private(self, body: dict[str, Any]) -> str:
    """Mint a new private stream from a create-request body; returns its name.

    Raises :class:`RuntimeError` when the private-stream cap is reached.
    """
    private = [n for n in self.states if n != DEFAULT_STREAM]
    if len(private) >= self.private_cap:
        raise RuntimeError(f"private stream cap reached ({self.private_cap})")
    self._private_seq += 1
    name = f"s{self._private_seq}"
    dw, dh = self.default_size
    self.create(
        name,
        width=int(body.get("width", dw)),
        height=int(body.get("height", dh)),
        adaptive=bool(body.get("adaptive", False)),
        still_after=_opt_float(body.get("still_after")),
        stats_interval=_opt_float(body.get("stats_interval"), default=self.stats_interval),
        encode_pipeline_depth=int(body.get("encode_pipeline_depth", 0)),
        resize_policy=str(body.get("resize_policy", "publisher")),
        max_render_dimension=_opt_int(body.get("max_render_dimension")),
        private=True,
    )
    return name

destroy(name) async

Cancel a private stream's publish task and remove it from the hub.

Source code in src/pdum/rfb/demo_server.py 
510
511
512
513
514
515
516
517
518
519
520
521
522
async def destroy(self, name: str) -> None:
    """Cancel a private stream's publish task and remove it from the hub."""
    if name == DEFAULT_STREAM:
        raise ValueError("cannot destroy the default stream")
    task = self.tasks.pop(name, None)
    if task is not None:
        task.cancel()
        with contextlib.suppress(asyncio.CancelledError):
            await task
    self.states.pop(name, None)
    self._empty_since.pop(name, None)
    self.server.remove_stream(name)
    log.info("stream %r destroyed", name)

available_backends()

(id, label) for every backend usable on this box (the greenlit subset).

Source code in src/pdum/rfb/demo_server.py 
136
137
138
def available_backends() -> list[tuple[str, str]]:
    """``(id, label)`` for every backend usable on this box (the greenlit subset)."""
    return [(b["id"], b["label"]) for b in backend_catalog() if b["available"]]

backend_catalog()

Every encode backend the demo knows, each tagged available/why-not for greying-out.

id is what :meth:~pdum.rfb.server._StreamHost.switch_backend accepts: image:<mode> or a registered video-encoder name. The image modes are always available; the video backends are gated by their runtime probe, and unavailable ones carry a short reason so the panel can explain the grey.

Source code in src/pdum/rfb/demo_server.py 
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
def backend_catalog() -> list[dict[str, Any]]:
    """Every encode backend the demo knows, each tagged available/why-not for greying-out.

    ``id`` is what :meth:`~pdum.rfb.server._StreamHost.switch_backend` accepts:
    ``image:<mode>`` or a registered video-encoder name. The image modes are always
    available; the video backends are gated by their runtime probe, and unavailable ones
    carry a short ``reason`` so the panel can explain the grey.
    """
    darwin = sys.platform == "darwin"
    have_av = importlib.util.find_spec("av") is not None
    entries: list[tuple[str, str, bool, str]] = [
        ("image:jpeg", "Image · JPEG (Pillow)", True, ""),
        ("image:png", "Image · PNG (lossless)", True, ""),
        ("image:webp", "Image · WebP (Pillow)", True, ""),
        ("h264_cpu", "H.264 · libx264 (CPU, PyAV)", have_av, "needs PyAV ([h264] extra)"),
        (
            "vtenc",
            "H.264 · VideoToolbox (Apple HW)",
            _probe("pdum.rfb.encoders.vtenc", "vtenc_available"),
            "macOS + Apple VideoToolbox only ([mac-vt] extra)",
        ),
        (
            "nvenc_cpu",
            "H.264 · NVENC (host input)",
            _probe("pdum.rfb.encoders.nvenc_cpu", "nvenc_cpu_available"),
            "needs an NVIDIA NVENC GPU + PyAV",
        ),
        (
            "nvenc_gpu_pyav",
            "H.264 · NVENC zero-copy (PyAV≥18)",
            _probe("pdum.rfb.gpu", "cuda_zerocopy_available"),
            "needs CuPy + PyAV≥18",
        ),
        (
            "nvenc_gpu_pdum",
            "H.264 · NVENC SDK (pdum.nvenc)",
            _probe("pdum.rfb.encoders.nvenc_gpu_pdum", "nvenc_gpu_pdum_available"),
            "needs habemus-papadum-nvenc + CUDA GPU",
        ),
    ]
    _ = darwin  # (reserved for future per-OS labels)
    return [{"id": i, "label": lbl, "available": ok, "reason": "" if ok else why} for i, lbl, ok, why in entries]

build_demo_app(*, width=1280, height=720, fps=30, bitrate='8M', stats_interval=1.0, static_dir=STATIC_DEMO_DIR, private_cap=8)

Build the demo's Starlette app: REST control + framebuffer WS + the static SPA.

The hub's websockets listener is intentionally never started — connections are driven through :func:~pdum.rfb.asgi.rfb_hub_endpoint, so everything shares one uvicorn origin. If static_dir is missing (SPA not built yet) a placeholder page is served so the control plane is still exercisable.

Source code in src/pdum/rfb/demo_server.py 
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
def build_demo_app(
    *,
    width: int = 1280,
    height: int = 720,
    fps: int = 30,
    bitrate: int | str = "8M",
    stats_interval: float | None = 1.0,
    static_dir: str | Path | None = STATIC_DEMO_DIR,
    private_cap: int = 8,
) -> Any:
    """Build the demo's Starlette app: REST control + framebuffer WS + the static SPA.

    The hub's ``websockets`` listener is intentionally never started — connections are
    driven through :func:`~pdum.rfb.asgi.rfb_hub_endpoint`, so everything shares one
    uvicorn origin. If ``static_dir`` is missing (SPA not built yet) a placeholder page is
    served so the control plane is still exercisable.
    """
    from starlette.applications import Starlette
    from starlette.requests import Request
    from starlette.responses import HTMLResponse, JSONResponse, Response
    from starlette.routing import Mount, Route, WebSocketRoute
    from starlette.staticfiles import StaticFiles

    from .asgi import rfb_hub_endpoint

    server = Server(host="127.0.0.1", port=0)  # port unused: no listener is started
    manager = DemoStreamManager(
        server,
        default_width=width,
        default_height=height,
        fps=fps,
        bitrate=_parse_bitrate(bitrate),
        stats_interval=stats_interval,
        private_cap=private_cap,
    )

    @contextlib.asynccontextmanager
    async def lifespan(app: Any):
        await manager.start()
        log.info("demo ready — %d scene(s), %d backend(s) available", len(available_demos()), len(available_backends()))
        try:
            yield
        finally:
            await manager.aclose()
            await server.aclose()

    def _host_or_404(name: str):
        host = server._streams.get(name)
        if host is None:
            return None
        return host

    async def caps_route(request: Request) -> JSONResponse:
        return JSONResponse(capabilities())

    async def state_route(request: Request) -> JSONResponse:
        return JSONResponse(manager.state())

    async def create_stream(request: Request) -> JSONResponse:
        body = await _json(request)
        try:
            name = manager.create_private(body)
        except RuntimeError as exc:
            return JSONResponse({"error": str(exc)}, status_code=429)
        return JSONResponse(manager.stream_state(name), status_code=201)

    async def delete_stream(request: Request) -> Response:
        name = request.path_params["name"]
        if name == DEFAULT_STREAM:
            return JSONResponse({"error": "cannot destroy the default stream"}, status_code=400)
        if name not in manager.states:
            return JSONResponse({"error": f"unknown stream {name!r}"}, status_code=404)
        await manager.destroy(name)
        return JSONResponse({"ok": True})

    async def set_scene(request: Request) -> JSONResponse:
        name = request.path_params["name"]
        host = _host_or_404(name)
        if host is None:
            return JSONResponse({"error": f"unknown stream {name!r}"}, status_code=404)
        body = await _json(request)
        key = body.get("key")
        try:
            manager.states[name].select(key)
        except KeyError:
            return JSONResponse({"error": f"unknown scene {key!r}"}, status_code=400)
        log.info("stream %r → scene %s", name, key)
        return JSONResponse(manager.stream_state(name))

    async def set_backend(request: Request) -> JSONResponse:
        name = request.path_params["name"]
        host = _host_or_404(name)
        if host is None:
            return JSONResponse({"error": f"unknown stream {name!r}"}, status_code=404)
        body = await _json(request)
        bid = body.get("id", "")
        if bid not in {b["id"] for b in backend_catalog()}:
            return JSONResponse({"error": f"unknown backend {bid!r}"}, status_code=400)
        try:
            host.switch_backend(bid)
        except Exception as exc:  # noqa: BLE001
            return JSONResponse({"error": f"backend switch failed: {exc}"}, status_code=400)
        log.info("stream %r → backend %s", name, bid)
        return JSONResponse(manager.stream_state(name))

    async def set_quality(request: Request) -> JSONResponse:
        name = request.path_params["name"]
        host = _host_or_404(name)
        if host is None:
            return JSONResponse({"error": f"unknown stream {name!r}"}, status_code=404)
        body = await _json(request)
        br = _parse_bitrate(body["bitrate"]) if body.get("bitrate") not in (None, "") else None
        fps = int(body["fps"]) if body.get("fps") not in (None, "") else None
        host.set_quality(bitrate=br, fps=fps)
        log.info("stream %r → quality bitrate=%s fps=%s", name, br, fps)
        return JSONResponse(manager.stream_state(name))

    async def set_params(request: Request) -> JSONResponse:
        """Apply the *live* stream params (resolution + color). Structural params are fixed
        at creation and are rejected here with a hint to make a private stream."""
        name = request.path_params["name"]
        host = _host_or_404(name)
        if host is None:
            return JSONResponse({"error": f"unknown stream {name!r}"}, status_code=404)
        body = await _json(request)
        st = manager.states[name]
        structural = {
            "adaptive",
            "still_after",
            "stats_interval",
            "encode_pipeline_depth",
            "resize_policy",
            "max_render_dimension",
        }
        rejected = structural & set(body)
        if rejected:
            return JSONResponse(
                {"error": f"structural params {sorted(rejected)} are set at stream creation — make a private stream"},
                status_code=409,
            )
        if "width" in body:
            st.width = _even(body["width"])
        if "height" in body:
            st.height = _even(body["height"])
        if "color" in body:
            st.color = body["color"] or None
        log.info("stream %r → params w=%s h=%s color=%s", name, st.width, st.height, st.color)
        return JSONResponse(manager.stream_state(name))

    async def metrics_route(request: Request) -> JSONResponse:
        host = _host_or_404(request.path_params["name"])
        if host is None:
            return JSONResponse([], status_code=404)
        return JSONResponse(host.metrics())

    async def placeholder(request: Request) -> HTMLResponse:
        return HTMLResponse(_PLACEHOLDER_HTML, status_code=200)

    routes: list[Any] = [
        Route("/demo/capabilities", caps_route),
        Route("/demo/state", state_route),
        Route("/demo/streams", create_stream, methods=["POST"]),
        Route("/demo/streams/{name}", delete_stream, methods=["DELETE"]),
        Route("/demo/streams/{name}/scene", set_scene, methods=["POST"]),
        Route("/demo/streams/{name}/backend", set_backend, methods=["POST"]),
        Route("/demo/streams/{name}/quality", set_quality, methods=["POST"]),
        Route("/demo/streams/{name}/params", set_params, methods=["POST"]),
        Route("/streams/{name}/metrics", metrics_route),
        WebSocketRoute("/rfb/{stream}", rfb_hub_endpoint(server)),
    ]
    sd = Path(static_dir) if static_dir else None
    if sd is not None and sd.is_dir():
        routes.append(Mount("/", app=StaticFiles(directory=str(sd), html=True)))
    else:
        routes.append(Route("/", placeholder))
        routes.append(Route("/{path:path}", placeholder))

    app = Starlette(lifespan=lifespan, routes=routes)
    app.state.manager = manager  # test hook
    app.state.server = server
    return app

capabilities()

The payload behind GET /demo/capabilities — drives greying-out + control render.

Source code in src/pdum/rfb/demo_server.py 
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
def capabilities() -> dict[str, Any]:
    """The payload behind ``GET /demo/capabilities`` — drives greying-out + control render."""
    mach = platform.machine()
    syst = platform.system()
    return {
        "scenes": scene_catalog(),
        "backends": backend_catalog(),
        "controls": _controls_schema(),
        "platform": {
            "system": syst,
            "machine": mach,
            "is_mac_arm": syst == "Darwin" and mach in ("arm64", "aarch64"),
            "python": ".".join(map(str, sys.version_info[:3])),
        },
        "limits": {"private_stream_cap": 8},
    }

make_dev_app()

App factory for pdum-rfb demo --dev — uvicorn reload=True re-imports this on every Python change, so config is read from env (it survives the reload subprocess). API-only: in dev the SPA is served by Vite (with HMR), which proxies REST + WS back to this process.

Source code in src/pdum/rfb/demo_server.py 
881
882
883
884
885
886
887
888
889
890
891
892
893
894
def make_dev_app() -> Any:
    """App factory for ``pdum-rfb demo --dev`` — uvicorn ``reload=True`` re-imports this on every
    Python change, so config is read from env (it survives the reload subprocess). API-only: in
    dev the SPA is served by Vite (with HMR), which proxies REST + WS back to this process."""
    import os

    _ensure_pdum_logging(logging.DEBUG if os.environ.get("RFB_DEMO_VERBOSE") == "1" else logging.INFO)
    return build_demo_app(
        width=int(os.environ.get("RFB_DEMO_W", "1280")),
        height=int(os.environ.get("RFB_DEMO_H", "720")),
        fps=int(os.environ.get("RFB_DEMO_FPS", "30")),
        bitrate=os.environ.get("RFB_DEMO_BITRATE", "8M"),
        static_dir=None,
    )

run_demo(*, width=1280, height=720, host='127.0.0.1', port=0, fps=30, bitrate='8M', verbose=False, open_browser=True, dev=False)

Serve the demo web app (blocking). Localhost-only by default; port=0 picks a free one.

open_browser launches the browser at the URL once it's up; dev runs the live-reload agentic mode (Vite HMR + uvicorn reload) — see :func:_run_dev.

Source code in src/pdum/rfb/demo_server.py 
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
def run_demo(
    *,
    width: int = 1280,
    height: int = 720,
    host: str = "127.0.0.1",
    port: int = 0,
    fps: int = 30,
    bitrate: int | str = "8M",
    verbose: bool = False,
    open_browser: bool = True,
    dev: bool = False,
) -> None:
    """Serve the demo web app (blocking). Localhost-only by default; ``port=0`` picks a free one.

    ``open_browser`` launches the browser at the URL once it's up; ``dev`` runs the live-reload
    agentic mode (Vite HMR + uvicorn reload) — see :func:`_run_dev`.
    """
    logging.basicConfig(
        level=logging.DEBUG if verbose else logging.INFO,
        format="%(asctime)s %(levelname)-5s %(name)s: %(message)s",
        datefmt="%H:%M:%S",
    )
    w, h = _even(width), _even(height)
    api_port = _free_port(host) if port == 0 else port
    if dev:
        _run_dev(host, api_port, w, h, fps, bitrate, open_browser, verbose)
    else:
        _run_static(host, api_port, w, h, fps, bitrate, open_browser)

scene_catalog()

Every built-in scene, tagged available/why-not for greying-out.

Source code in src/pdum/rfb/demo_server.py 
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def scene_catalog() -> list[dict[str, Any]]:
    """Every built-in scene, tagged available/why-not for greying-out."""
    out: list[dict[str, Any]] = []
    for d in DEMOS:
        try:
            ok = bool(d.available())
        except Exception:
            ok = False
        reason = "" if ok else "unavailable on this platform (missing deps/hardware)"
        out.append(
            {
                "key": d.key,
                "name": d.name,
                "description": d.description,
                "tags": list(d.tags),
                "available": ok,
                "reason": reason,
            }
        )
    return out

smoke(*, width=320, height=240, fps=30, verbose=True)

Drive the real ASGI demo in-process: capabilities, every backend over REST on one socket, a 2-viewer fan-out check, and a private-stream create→connect→destroy cycle.

Uses Starlette's TestClient (no uvicorn, no real port, no browser). Returns a result dict; raises AssertionError on any failure. This is the CI-grade proof.

Source code in src/pdum/rfb/demo_server.py 
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
def smoke(*, width: int = 320, height: int = 240, fps: int = 30, verbose: bool = True) -> dict:
    """Drive the real ASGI demo in-process: capabilities, every backend over REST on one
    socket, a 2-viewer fan-out check, and a private-stream create→connect→destroy cycle.

    Uses Starlette's ``TestClient`` (no uvicorn, no real port, no browser). Returns a
    result dict; raises ``AssertionError`` on any failure. This is the CI-grade proof.
    """
    from starlette.testclient import TestClient

    from .protocol import unpack_binary_message
    from .testing import decode_annexb

    def _log(msg: str) -> None:
        if verbose:
            print(f"[demo smoke] {msg}")

    w, h = _even(width), _even(height)
    app = build_demo_app(width=w, height=h, fps=fps, bitrate="4M", static_dir=None)
    results: dict[str, Any] = {"backends": {}}

    def _drain_until(ws: Any, want_kind: str, budget: int = 240) -> dict:
        for _ in range(budget):
            msg = ws.receive()
            if msg.get("type") == "websocket.close":
                raise AssertionError("socket closed while draining")
            data = msg.get("bytes")
            if data is None:
                continue
            header, payload = unpack_binary_message(data)
            ws.send_json({"type": "ack", "seq": header["seq"], "decode_queue_size": 0})
            kind = "video" if header["type"] == "video_chunk" else "image"
            if kind == want_kind:
                return {"header": header, "payload": payload}
        raise AssertionError(f"never saw a {want_kind} frame")

    with TestClient(app) as client:
        caps = client.get("/demo/capabilities").json()
        backends = [b for b in caps["backends"] if b["available"]]
        _log(f"scenes: {[s['key'] for s in caps['scenes'] if s['available']]}")
        _log(f"backends: {[b['id'] for b in backends]}")
        results["scenes"] = [s["key"] for s in caps["scenes"] if s["available"]]

        with client.websocket_connect("/rfb/default") as ws:
            ws.send_json({"type": "hello", "supported": _ALL_CAPS, "device_pixel_ratio": 1})
            config = ws.receive_json()
            assert config["type"] == "config", config
            _log(f"connected; initial transport={config['transport']}")

            for b in backends:
                bid = b["id"]
                want = "image" if bid.startswith("image:") else "video"
                r = client.post("/demo/streams/default/backend", json={"id": bid})
                assert r.status_code == 200, r.text
                got = _drain_until(ws, want)
                header, payload = got["header"], got["payload"]
                assert header["width"] == w and header["height"] == h, header
                if want == "video":
                    decode_annexb(payload)  # must not raise
                    detail = f"codec={header.get('codec')} bytes={len(payload)}"
                else:
                    from io import BytesIO

                    from PIL import Image

                    img = Image.open(BytesIO(payload))
                    img.load()
                    assert (img.width, img.height) == (w, h)
                    detail = f"mime={header.get('mime')} {img.width}x{img.height}"
                results["backends"][bid] = detail
                _log(f"  ✓ {bid}: {detail}")

            # Live retune over REST.
            assert client.post("/demo/streams/default/quality", json={"bitrate": "2M", "fps": 20}).status_code == 200
            _drain_until(ws, "image" if backends[-1]["id"].startswith("image:") else "video")
            _log("  ✓ REST quality retune — stream continued")

            # Scene switch + input round-trip on the paint demo.
            if any(s["key"] == "paint" for s in caps["scenes"]):
                assert client.post("/demo/streams/default/scene", json={"key": "paint"}).status_code == 200
                ws.send_json({"type": "event", "event": {"type": "pointer_down", "x": 5, "y": 5, "buttons": [1]}})
                _drain_until(ws, "image" if backends[-1]["id"].startswith("image:") else "video")
                results["scene_switch"] = True
                _log("  ✓ REST scene switch → paint + pointer event")

            # Multi-client fan-out: a second viewer on the same shared stream.
            with client.websocket_connect("/rfb/default") as ws2:
                ws2.send_json({"type": "hello", "supported": _ALL_CAPS, "device_pixel_ratio": 1})
                assert ws2.receive_json()["type"] == "config"
                _drain_until(ws2, "image" if backends[-1]["id"].startswith("image:") else "video")
                assert app.state.manager.stream_state("default")["clients"] >= 2
                results["fanout"] = True
                _log("  ✓ 2-viewer fan-out on the shared stream")

        # Private stream: create → connect → destroy.
        created = client.post("/demo/streams", json={"width": w, "height": h})
        assert created.status_code == 201, created.text
        pname = created.json()["name"]
        client.post(f"/demo/streams/{pname}/backend", json={"id": "image:jpeg"})  # deterministic
        with client.websocket_connect(f"/rfb/{pname}") as pws:
            pws.send_json({"type": "hello", "supported": _ALL_CAPS, "device_pixel_ratio": 1})
            assert pws.receive_json()["type"] == "config"
            _drain_until(pws, "image")  # default backend on a fresh private stream is image/auto
        assert client.delete(f"/demo/streams/{pname}").status_code == 200
        assert pname not in app.state.manager.states
        results["private_stream"] = pname
        _log(f"  ✓ private stream {pname} create → connect → destroy")

    _log("ALL GREEN")
    results["ok"] = True
    return results

demos

Built-in demo scenes for pdum-rfb demo (the interactive harness).

Each :class:Demo is a small, self-contained scene the harness can publish into a shared :class:~pdum.rfb.display.Display. A demo is a factory (make()) so selecting it starts fresh; the resulting instance exposes:

  • frame(seq, t, width, height) -> np.ndarray — the RGB(A) frame to publish, and
  • optionally on_event(event) -> None — to consume browser input (pointer/key/wheel).

Adding a demo is a few lines: write a make returning an object with frame (and maybe on_event), then append a :class:Demo to :data:DEMOS. Demos whose available() returns False (missing platform/deps) are hidden by the harness.

Demo dataclass

A selectable demo scene.

Source code in src/pdum/rfb/demos.py 
30
31
32
33
34
35
36
37
38
39
@dataclass(slots=True)
class Demo:
    """A selectable demo scene."""

    key: str
    name: str
    description: str
    make: Callable[[], Any]  # () -> instance with .frame(seq, t, w, h) [+ .on_event(ev)]
    available: Callable[[], bool] = field(default=lambda: True)
    tags: tuple[str, ...] = ()  # e.g. ("cpu",), ("mlx", "metal"), ("interactive",)

available_demos()

The demos viable on this machine (filters out unavailable platform/deps).

Source code in src/pdum/rfb/demos.py 
263
264
265
def available_demos() -> list[Demo]:
    """The demos viable on this machine (filters out unavailable platform/deps)."""
    return [d for d in DEMOS if _safe(d.available)]

display

The push-model :class:Display: you publish frames, viewers attach to watch.

The application owns its loop and pushes the latest frame into a Display; the library fans that frame out to every connected browser, each driven by its own :class:~pdum.rfb.session.RfbSession and encoder (so each viewer gets a keyframe on attach and independent latest-frame-wins backpressure). Input events from all viewers funnel into one stream the application drains with :meth:poll_events.

display = await rfb.serve(1280, 720, port=8765)   # background WS server + handle
state = init()
while running:
    for ev in display.poll_events():     # ev.client_id, ev.principal, ev.event
        state = update(state, ev)
    display.publish(render(state))       # sync, non-blocking, latest-wins
    await asyncio.sleep(1 / 30)          # or ad-hoc / every 60 s — you own the cadence
await display.aclose()

publish() must be called on the event-loop thread (it wakes feeds via asyncio.Event). Publishing a differently-shaped array transparently rebuilds each viewer's encoder and forces a keyframe; keep pixel_format constant.

Display

A single shared framebuffer that one or more browsers attach to.

Parameters:

Name Type Description Default
width int

Initial framebuffer size. Updated automatically whenever you publish a differently-shaped frame.

 required
height int

Initial framebuffer size. Updated automatically whenever you publish a differently-shaped frame.

 required
fps int

Advisory frame rate (used as the encoder's IDR cadence / metrics target); the actual cadence is whatever your publish loop does.

 30
record_events bool

Also accumulate raw events in :attr:recorded (exposed via the server's GET /recorded-events side channel and the headless e2e harness).

 False
event_log str | Path | None

Optional path; received events are appended as JSON lines.

 None
event_queue_size int

Bound on the un-polled event backlog; the oldest events are dropped when a publisher never calls :meth:poll_events.

 4096
own_frames bool

Opt in to server-owned frames. By default :meth:publish borrows the caller's buffer (zero-copy) and reads it asynchronously, so you must publish a fresh buffer each call (or not mutate it until it is encoded). With own_frames=True each published frame is copied into a server-owned, recycled buffer on the publish thread, so you may reuse/mutate your own buffer immediately after :meth:publish returns — no reallocation and no "frame released" callback. Supported for cpu and cuda frames; metal raises (MLX arrays are immutable, so the borrow contract already holds). See :meth:publish.

 False
resize_policy str

"publisher" (default) — you own the render size and a viewer's set_viewport is informational. "match_client" — the render stream follows the viewer: the latest set_viewport becomes :attr:target_size (last-writer-wins across viewers), which your render loop reads to size the next frame.

 'publisher'
max_render_dimension int | None

Cap on either dimension of a match_client :attr:target_size (AR-preserving), guarding against a maximized 4K window forcing a huge encode. None = no cap.

 None
resize_debounce float

Seconds a match_client target must be stable before it surfaces through :attr:target_size, so a drag-resize doesn't storm the encoder rebuild (default 0.12).

 0.12
clock Callable[[], float] | None

Monotonic clock returning seconds; injectable for deterministic tests.

 None
Source code in src/pdum/rfb/display.py 
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
class Display:
    """A single shared framebuffer that one or more browsers attach to.

    Parameters
    ----------
    width, height:
        Initial framebuffer size. Updated automatically whenever you publish a
        differently-shaped frame.
    fps:
        Advisory frame rate (used as the encoder's IDR cadence / metrics target);
        the *actual* cadence is whatever your publish loop does.
    record_events:
        Also accumulate raw events in :attr:`recorded` (exposed via the server's
        ``GET /recorded-events`` side channel and the headless e2e harness).
    event_log:
        Optional path; received events are appended as JSON lines.
    event_queue_size:
        Bound on the un-polled event backlog; the **oldest** events are dropped
        when a publisher never calls :meth:`poll_events`.
    own_frames:
        Opt in to **server-owned frames**. By default :meth:`publish` *borrows* the
        caller's buffer (zero-copy) and reads it asynchronously, so you must publish a
        fresh buffer each call (or not mutate it until it is encoded). With
        ``own_frames=True`` each published frame is **copied into a server-owned,
        recycled buffer** on the publish thread, so you may reuse/mutate your own buffer
        immediately after :meth:`publish` returns — no reallocation and no "frame
        released" callback. Supported for ``cpu`` and ``cuda`` frames; ``metal`` raises
        (MLX arrays are immutable, so the borrow contract already holds). See
        :meth:`publish`.
    resize_policy:
        ``"publisher"`` (default) — you own the render size and a viewer's ``set_viewport``
        is informational. ``"match_client"`` — the render stream *follows the viewer*: the
        latest ``set_viewport`` becomes :attr:`target_size` (last-writer-wins across viewers),
        which your render loop reads to size the next frame.
    max_render_dimension:
        Cap on either dimension of a ``match_client`` :attr:`target_size` (AR-preserving),
        guarding against a maximized 4K window forcing a huge encode. ``None`` = no cap.
    resize_debounce:
        Seconds a ``match_client`` target must be stable before it surfaces through
        :attr:`target_size`, so a drag-resize doesn't storm the encoder rebuild (default 0.12).
    clock:
        Monotonic clock returning seconds; injectable for deterministic tests.
    """

    def __init__(
        self,
        width: int,
        height: int,
        *,
        fps: int = 30,
        record_events: bool = False,
        event_log: str | Path | None = None,
        event_queue_size: int = 4096,
        own_frames: bool = False,
        resize_policy: str = "publisher",
        max_render_dimension: int | None = None,
        resize_debounce: float = 0.12,
        clock: Callable[[], float] | None = None,
    ) -> None:
        self.width = int(width)
        self.height = int(height)
        # The initial (native/full-resolution) size, kept fixed even as publish() resizes
        # width/height. It is the base a DownscaleHint scales off, so an absolute scale
        # never compounds when the render loop honors the hint by publishing smaller.
        self._base_width = int(width)
        self._base_height = int(height)
        self.fps = fps
        self._clock = clock or time.monotonic
        self._start = self._clock()

        # "match-client" resize policy: when enabled, a viewer's set_viewport becomes a
        # *target size* the render loop follows (default "publisher" = you own the size,
        # set_viewport is informational). Last-writer-wins across viewers; debounced so a
        # drag-resize doesn't storm the encoder rebuild; clamped to max_render_dimension.
        if resize_policy not in ("publisher", "match_client"):
            raise ValueError(f"resize_policy must be 'publisher' or 'match_client', got {resize_policy!r}")
        self.resize_policy = resize_policy
        self.max_render_dimension = max_render_dimension
        self._resize_debounce = float(resize_debounce)
        self._pending_target: tuple[int, int] | None = None
        self._pending_ratio = 1.0
        self._pending_at = 0.0
        self._committed_target: tuple[int, int] | None = None
        self._committed_ratio = 1.0

        self._latest: RawFrame | None = None
        self._version = 0
        # Opt-in frame ownership (own_frames=True): publish() copies each frame into a
        # server-owned buffer drawn from this recycled pool, so the caller may reuse its
        # buffer immediately. Empty/unused when own_frames is False. See _own_copy / _take_owned.
        self._own_frames = bool(own_frames)
        self._own_pool: list[Any] = []
        self._own_key: tuple[Any, ...] | None = None
        self._feeds: set[_ClientFeed] = set()
        self._sessions: set[RfbSession] = set()
        self._clients: dict[str, _ClientFeed] = {}
        # Frame taps that are *not* viewers: server-side recordings (see record()). They are
        # woken by publish() like feeds but excluded from client_count and the adaptive
        # downscale aggregation.
        self._taps: set[Any] = set()

        # Adaptive resolution (serve(adaptive=True)): the effective render scale is the
        # minimum any connected viewer's controller currently wants (the most-congested
        # viewer wins); a change enqueues a DownscaleHint through poll_events().
        self._effective_scale = 1.0

        self._events: deque[InputEvent | DownscaleHint] = deque(maxlen=event_queue_size)
        self._events_signal = asyncio.Event()

        self._record_events = record_events or event_log is not None
        self._event_log = Path(event_log) if event_log else None
        self.recorded: list[EventDict] = []

        self._closed = False
        # Set by serve() so aclose() can stop the listener it started.
        self._server: Any = None
        self._server_cm: Any = None
        # Set by Server.add_stream() so display.server.add_stream(...) works.
        self._owner_server: Any = None
        # The URL path segment this stream is reached at (set by Server.add_stream()).
        self._stream_name: str = "default"

    # --- publishing --------------------------------------------------------

    def publish(
        self,
        frame: np.ndarray | RawFrame | Any,
        *,
        pixel_ratio: float | None = None,
        color: Any = None,
    ) -> None:
        """Make ``frame`` the latest frame and wake every connected viewer.

        Synchronous and non-blocking. ``frame`` may be:

        * a contiguous host ``uint8`` array — ``(H, W, 3)`` ``rgb24`` or
          ``(H, W, 4)`` ``rgba8``;
        * a **CUDA tensor** exposing ``__cuda_array_interface__`` (e.g. CuPy) of
          shape ``(H, W, 3|4)`` — published as a zero-copy ``cuda`` frame (for
          NV12, or other frameworks, build a ``RawFrame`` via
          :func:`pdum.rfb.gpu.cuda_frame`);
        * an **MLX (Apple Metal) array** of shape ``(H, W, 3|4)`` — published as a
          ``metal`` frame; the VideoToolbox encoder converts RGB(A)→NV12 on the GPU
          (for pre-converted NV12, use :func:`pdum.rfb.metal.metal_frame`);
        * a ready :class:`~pdum.rfb.types.RawFrame` (any ``memory``).

        Latest-frame-wins: a viewer that is behind simply skips intermediate frames.

        **Ownership.** By default ``publish()`` *borrows* your buffer — it stores a bare
        reference and reads the pixels **asynchronously**, on each viewer's encode worker
        thread. The borrow window runs from here until every viewer has finished encoding
        the frame, and it is **widest under** ``still_after`` (the resting frame is re-read
        ~``still_after`` seconds later for the lossless still). So in borrow mode, publish a
        *fresh* buffer each call, or do not mutate a published buffer until it is encoded.
        Construct the ``Display`` (or call :func:`~pdum.rfb.serve`) with ``own_frames=True``
        to instead have the server copy each frame into a recycled buffer, after which you
        may reuse/mutate your own buffer immediately (``cpu``/``cuda`` only; ``metal`` raises).

        Parameters
        ----------
        pixel_ratio:
            Render-side DPR for this frame (device px per logical px). ``None`` keeps a
            :class:`~pdum.rfb.types.RawFrame`'s own value (else ``1.0``). See
            :attr:`~pdum.rfb.types.RawFrame.pixel_ratio`.
        color:
            Color descriptor for this frame — a :class:`~pdum.rfb.types.ColorSpace`, its
            ``dict`` form, or ``None`` (sRGB). The renderer must already produce pixels in
            the declared space; the library only tags them.
        """
        if self._closed:
            raise RuntimeError("publish() called on a closed Display")

        frame_pr = frame.pixel_ratio if isinstance(frame, RawFrame) else 1.0
        frame_color = frame.color if isinstance(frame, RawFrame) else None
        resolved_pr = float(frame_pr if pixel_ratio is None else pixel_ratio)
        resolved_color = frame_color if color is None else _color_to_dict(color)

        if isinstance(frame, RawFrame):
            data, width, height = frame.data, frame.width, frame.height
            pixel_format, memory = frame.pixel_format, frame.memory
        elif isinstance(frame, np.ndarray):
            if frame.ndim != 3 or frame.shape[2] not in (3, 4):
                raise ValueError(f"unsupported frame shape {frame.shape!r}; expected (H, W, 3) or (H, W, 4)")
            height, width = int(frame.shape[0]), int(frame.shape[1])
            pixel_format = "rgb24" if frame.shape[2] == 3 else "rgba8"
            data, memory = frame, "cpu"
        elif _is_cuda_tensor(frame):
            shape = getattr(frame, "shape", None)
            if shape is None or len(shape) != 3 or shape[2] not in (3, 4):
                raise ValueError("CUDA publish expects an (H, W, 3|4) tensor; use pdum.rfb.gpu.cuda_frame() for NV12")
            height, width = int(shape[0]), int(shape[1])
            pixel_format = "rgb24" if shape[2] == 3 else "rgba8"
            data, memory = frame, "cuda"
        elif _is_metal_tensor(frame):
            shape = getattr(frame, "shape", None)
            if shape is None or len(shape) != 3 or shape[2] not in (3, 4):
                raise ValueError(
                    "Metal publish expects an (H, W, 3|4) MLX array; use pdum.rfb.metal.metal_frame() for NV12"
                )
            height, width = int(shape[0]), int(shape[1])
            pixel_format = "rgb24" if shape[2] == 3 else "rgba8"
            data, memory = frame, "metal"
        else:
            raise TypeError("publish() expects a numpy.ndarray, a CUDA/Metal tensor, or a RawFrame")

        if memory == "metal":
            # Materialize the lazy MLX render on *this* (loop) thread: MLX binds a lazy graph to
            # its origin thread's stream, so the session's encode worker thread cannot evaluate a
            # frame built here. The GPU NV12 conversion still runs on the worker. See metal.materialize.
            from .metal import materialize

            materialize(data)

        if self._own_frames:
            # Server-owned mode: copy into a recycled buffer on this (loop) thread so the caller
            # may reuse/mutate its own buffer immediately. Severs the async-read aliasing entirely.
            data = self._own_copy(data, memory)

        timestamp_us = int((self._clock() - self._start) * 1_000_000)
        # seq is a placeholder; each feed stamps its own per-client sequence.
        self._latest = RawFrame(
            seq=0,
            width=width,
            height=height,
            timestamp_us=timestamp_us,
            pixel_format=pixel_format,  # type: ignore[arg-type]
            memory=memory,  # type: ignore[arg-type]
            data=data,
            pixel_ratio=resolved_pr,
            color=resolved_color,
        )
        self.width, self.height = width, height
        self._version += 1
        for feed in self._feeds:
            feed._wake()
        for tap in self._taps:
            tap._wake()

    def _own_copy(self, data: Any, memory: str) -> Any:
        """Copy ``data`` into a server-owned buffer (``own_frames`` mode) so the caller may
        reuse its own buffer immediately. Runs on the loop thread. ``cpu`` → numpy, ``cuda`` →
        CuPy device-to-device; ``metal`` is unsupported (MLX is immutable — borrow already holds)."""
        if memory == "cpu":
            arr = np.asarray(data)
            buf = self._take_owned(arr.shape, arr.dtype, memory)
            np.copyto(buf, arr)
            return buf
        if memory == "cuda":
            import cupy as cp  # lazy: only when a cuda frame is published under own_frames

            src = cp.asarray(data)
            buf = self._take_owned(src.shape, src.dtype, memory)
            cp.copyto(buf, src)
            return buf
        raise NotImplementedError(
            "own_frames is not supported for Metal frames; publish a fresh mx.array per frame "
            "(MLX arrays are immutable, so the borrow contract already holds). See docs/metal_videotoolbox.md."
        )

    def _take_owned(self, shape: tuple[int, ...], dtype: Any, memory: str) -> Any:
        """Return a reusable server-owned buffer of ``(shape, dtype, memory)`` that no in-flight
        frame still references. A size/dtype/memory change drops the pool (reallocate, like the
        encoder rebuild on resize).

        Correctness rests on CPython refcounting: while a session holds ``frame =
        replace(latest, seq)`` across its off-thread encode, that buffer's refcount is elevated,
        so it is skipped here and never overwritten mid-encode. Steady state the pool stabilizes
        at ~(concurrently-encoding viewers + 1) buffers and recycles them — no per-frame alloc."""
        key = (shape, dtype, memory)
        if key != self._own_key:
            self._own_pool = []
            self._own_key = key
        pool = self._own_pool
        for i in range(len(pool)):
            # getrefcount == 2 means the only refs are the pool slot + getrefcount's own argument,
            # i.e. nothing else holds it. Index as pool[i] (no bound local), or a temporary local
            # would add one and mask a free buffer.
            if sys.getrefcount(pool[i]) <= 2:
                return pool[i]
        if memory == "cuda":
            import cupy as cp

            buf = cp.empty(shape, dtype)
        else:
            buf = np.empty(shape, dtype)  # C-contiguous regardless of caller layout
        pool.append(buf)
        return buf

    # --- recording ---------------------------------------------------------

    def record(
        self,
        path: str | Path,
        *,
        fps: int | None = None,
        bitrate: int | None = None,
        crf: int | None = None,
        preset: str = "veryfast",
    ) -> Recording:
        """Record the published-frame stream to a real **MP4** file (server-side tap).

        Returns a started :class:`~pdum.rfb.recording.Recording` — an async context
        manager / handle with :meth:`~pdum.rfb.recording.Recording.stop`. It taps the
        same latest-frame stream the viewer sessions pull from, so it is **independent
        of any connected browser** and works fully headless (demos, CI artifacts,
        golden-frame capture). Frames are H.264-encoded (libx264) and muxed to
        **AVCC-in-MP4** via a **separate** PyAV path — this is the one place AVCC/MP4 is
        correct; it never touches the Annex-B WebSocket payload.

        The real, monotonic per-frame timestamps are honored, so a variable-cadence
        publisher (sparse ``still_after`` scenes included) records with correct frame
        timing. Call it on the event-loop thread; it starts a background encode task.

        ::

            async with display.record("out.mp4"):
                for _ in range(300):
                    display.publish(render()); await asyncio.sleep(1 / 30)
            # file finalized on exit — or use rec = display.record(...); await rec.stop()

        Parameters
        ----------
        path:
            Output ``.mp4`` file path.
        fps:
            Advisory frame rate for the encoder's rate control (default: the display's
            ``fps``). The *actual* timing comes from the frames' real timestamps.
        bitrate:
            Target bitrate in bits/s (average). Mutually exclusive with ``crf``; if
            neither is given, a sensible constant-quality ``crf`` is used.
        crf:
            Constant Rate Factor (0–51, lower = better) for constant-quality encoding.
        preset:
            libx264 speed/quality preset (default ``"veryfast"``).

        Raises
        ------
        RuntimeError
            If PyAV (the ``[h264]`` extra) is not installed.
        """
        from .recording import Recording

        rec = Recording(self, path, fps=fps or self.fps, bitrate=bitrate, crf=crf, preset=preset)
        rec._start_task()
        return rec

    # --- events ------------------------------------------------------------

    def poll_events(self) -> list[InputEvent | DownscaleHint]:
        """Drain and return everything queued since the last poll.

        The list is ordinarily :class:`~pdum.rfb.types.InputEvent`\\ s (input from all
        connected viewers). With ``serve(adaptive=True)`` it may also carry a
        :class:`~pdum.rfb.types.DownscaleHint` — a server-driven resolution suggestion —
        interleaved in arrival order; tell them apart with ``isinstance``.
        """
        out = list(self._events)
        self._events.clear()
        return out

    async def events(self) -> AsyncIterator[InputEvent | DownscaleHint]:
        """Async-iterate queued events (alternative to :meth:`poll_events`).

        Yields :class:`~pdum.rfb.types.InputEvent`\\ s and, under
        ``serve(adaptive=True)``, :class:`~pdum.rfb.types.DownscaleHint`\\ s. Use this or
        :meth:`poll_events` — both drain the same queue.
        """
        while not self._closed:
            if self._events:
                yield self._events.popleft()
                continue
            self._events_signal.clear()
            await self._events_signal.wait()

    @property
    def client_count(self) -> int:
        """Number of currently connected viewers."""
        return len(self._feeds)

    @property
    def pixel_ratio(self) -> float:
        """Render-side DPR of the latest published frame (``1.0`` before the first publish)."""
        return self._latest.pixel_ratio if self._latest is not None else 1.0

    @property
    def color(self) -> dict | None:
        """Color descriptor of the latest published frame, or ``None`` (sRGB)."""
        return self._latest.color if self._latest is not None else None

    # --- match-client resize (opt-in) --------------------------------------

    def _clamp_render_size(self, w: int, h: int) -> tuple[int, int]:
        """Clamp a requested render size to ``max_render_dimension`` (AR-preserving) and to
        even, >= 2 dimensions (H.264 / NV12 need even). ``max_render_dimension=None`` = no cap."""
        w, h = max(2, int(w)), max(2, int(h))
        cap = self.max_render_dimension
        if cap is not None and max(w, h) > cap:
            scale = cap / max(w, h)
            w, h = max(2, round(w * scale)), max(2, round(h * scale))
        return (w - (w % 2), h - (h % 2))

    def _request_target(self, pw: int, ph: int, ratio: float) -> None:
        """Record a viewer's requested render size (``match_client`` only). Debounced: the
        value surfaces through :attr:`target_size` after it has been stable for
        ``resize_debounce`` seconds, so a drag-resize doesn't storm the encoder rebuild."""
        size = self._clamp_render_size(pw, ph)
        if size != self._pending_target:
            self._pending_target = size
            self._pending_ratio = float(ratio)
            self._pending_at = self._clock()

    def _settle_target(self) -> None:
        if self._pending_target is not None and (self._clock() - self._pending_at) >= self._resize_debounce:
            self._committed_target = self._pending_target
            self._committed_ratio = self._pending_ratio
            self._pending_target = None

    @property
    def target_size(self) -> tuple[int, int] | None:
        """Latest client-requested render size under ``resize_policy="match_client"`` (debounced,
        clamped, even dims), or ``None`` before any viewport arrives / in ``"publisher"`` mode.

        Read it in your render loop to follow the viewer::

            w, h = display.target_size or (display.width, display.height)
            display.publish(render(state, w, h), pixel_ratio=display.target_ratio)
        """
        self._settle_target()
        return self._committed_target

    @property
    def target_ratio(self) -> float:
        """The client DPR that accompanies :attr:`target_size` (``1.0`` until one arrives)."""
        self._settle_target()
        return self._committed_ratio

    @property
    def port(self) -> int | None:
        """The bound TCP port (useful when serving with ``port=0``)."""
        if self._server is None:
            return None
        return next(iter(self._server.sockets)).getsockname()[1]

    @property
    def server(self) -> Any:
        """The owning :class:`~pdum.rfb.server.Server` hub, if this is a stream of one.

        Lets the convenience ``display = await serve(...)`` path reach the hub to add
        more streams: ``display.server.add_stream("camera_b", 640, 480)``. ``None``
        for a bare ``Display`` constructed directly.
        """
        return self._owner_server

    @property
    def ws_url(self) -> str:
        """Browser-reachable WebSocket URL for this stream (e.g. ``ws://host:port/name``).

        Raises if the server is not bound yet (call ``await rfb.serve(...)`` first). A
        wildcard bind host (``0.0.0.0``/``::``) is reported as ``127.0.0.1``.
        """
        if self.port is None:
            raise RuntimeError("Display is not serving yet; call await rfb.serve(...) first")
        host = getattr(self._owner_server, "host", None) or "127.0.0.1"
        if host in ("0.0.0.0", "", "::"):
            host = "127.0.0.1"
        return f"ws://{host}:{self.port}/{self._stream_name}"

    def widget(
        self,
        *,
        batteries: bool = True,
        base_path: str | None = None,
        host: str | None = None,
        **chrome: Any,
    ) -> Any:
        """Return an anywidget viewer bound to this stream (needs the ``[anywidget]`` extra).

        In a notebook: ``display.widget()`` renders the batteries viewer;
        ``display.widget(batteries=False)`` the bare canvas. One widget = one Web Worker +
        one WebSocket; the server multiplexes many streams on one port. For remote/HTTPS
        notebooks, mount ``pdum.rfb.asgi.rfb_hub_endpoint`` same-origin and pass
        ``base_path=`` (the widget then uses a same-origin ``wss://`` URL). Extra keyword
        args (e.g. ``show_toolbar=False``) become widget traits.

        Raises if the server is not bound yet.
        """
        from .notebook import RfbCanvas, RfbViewer

        if self.port is None:
            raise RuntimeError("Display is not serving yet; call await rfb.serve(...) first")
        resolved_host = host if host is not None else (getattr(self._owner_server, "host", None) or "127.0.0.1")
        if resolved_host in ("0.0.0.0", "", "::"):
            # Let the browser use the page's own hostname (correct for remote notebooks).
            resolved_host = "auto"
        cls = RfbViewer if batteries else RfbCanvas
        kwargs: dict[str, Any] = {"port": self.port, "stream": self._stream_name, "host": resolved_host, **chrome}
        if base_path is not None:
            kwargs["base_path"] = base_path
        return cls(**kwargs)

    # --- lifecycle ---------------------------------------------------------

    def _close_local(self) -> None:
        """Disconnect viewers and wake waiters, *without* stopping any listener.

        The per-stream half of teardown: a hub (:class:`~pdum.rfb.server.Server`)
        calls this on each stream while it stops the shared listener once.
        """
        if self._closed:
            return
        self._closed = True
        for feed in list(self._feeds):
            feed.close()
        for tap in list(self._taps):
            tap.close()
        self._events_signal.set()

    async def aclose(self) -> None:
        """Stop the server, disconnect viewers, and release encoder resources."""
        self._close_local()
        if self._server_cm is not None:
            cm, self._server_cm = self._server_cm, None
            self._server = None
            await cm.__aexit__(None, None, None)

    # --- internal (used by the connection server) --------------------------

    def _enqueue_event(self, client_id: str, principal: Principal | None, event: EventDict) -> None:
        received_us = int((self._clock() - self._start) * 1_000_000)
        self._events.append(InputEvent(client_id=client_id, principal=principal, event=event, received_us=received_us))
        self._events_signal.set()
        if self._record_events:
            self.recorded.append(event)
            if self._event_log is not None:
                with self._event_log.open("a") as fh:
                    fh.write(json.dumps(event) + "\n")

    def _add_tap(self, tap: Any) -> None:
        """Register a non-viewer frame tap (e.g. a :class:`Recording`); publish() wakes it."""
        self._taps.add(tap)

    def _remove_tap(self, tap: Any) -> None:
        self._taps.discard(tap)

    def _recompute_render_hint(self) -> None:
        """Recompute the aggregate adaptive render scale and enqueue a
        :class:`~pdum.rfb.types.DownscaleHint` when it changes.

        The effective scale is the **minimum** any connected viewer's controller wants,
        so one congested viewer downscales the shared stream for everyone; when that
        viewer recovers or disconnects the scale climbs back and a recovery hint fires.
        No-op unless the value actually changed (de-duped, so it never spams the queue).
        """
        effective = min((f.render_scale for f in self._feeds), default=1.0)
        if effective == self._effective_scale:
            return
        self._effective_scale = effective
        w = max(2, round(self._base_width * effective))
        h = max(2, round(self._base_height * effective))
        w -= w % 2
        h -= h % 2
        received_us = int((self._clock() - self._start) * 1_000_000)
        self._events.append(DownscaleHint(scale=effective, width=w, height=h, received_us=received_us))
        self._events_signal.set()

    def _make_feed(self, client_id: str, principal: Principal | None) -> _ClientFeed:
        feed = _ClientFeed(self, client_id, principal)
        self._feeds.add(feed)
        self._clients[client_id] = feed
        return feed

    def _register_session(self, session: RfbSession) -> None:
        self._sessions.add(session)

    def _remove(self, client_id: str, feed: _ClientFeed, session: RfbSession | None) -> None:
        self._feeds.discard(feed)
        self._clients.pop(client_id, None)
        if session is not None:
            self._sessions.discard(session)
        # A viewer leaving can relax the aggregate downscale (its congestion is gone).
        self._recompute_render_hint()

client_count property

Number of currently connected viewers.

color property

Color descriptor of the latest published frame, or None (sRGB).

pixel_ratio property

Render-side DPR of the latest published frame (1.0 before the first publish).

port property

The bound TCP port (useful when serving with port=0).

server property

The owning :class:~pdum.rfb.server.Server hub, if this is a stream of one.

Lets the convenience display = await serve(...) path reach the hub to add more streams: display.server.add_stream("camera_b", 640, 480). None for a bare Display constructed directly.

target_ratio property

The client DPR that accompanies :attr:target_size (1.0 until one arrives).

target_size property

Latest client-requested render size under resize_policy="match_client" (debounced, clamped, even dims), or None before any viewport arrives / in "publisher" mode.

Read it in your render loop to follow the viewer::

w, h = display.target_size or (display.width, display.height)
display.publish(render(state, w, h), pixel_ratio=display.target_ratio)

ws_url property

Browser-reachable WebSocket URL for this stream (e.g. ws://host:port/name).

Raises if the server is not bound yet (call await rfb.serve(...) first). A wildcard bind host (0.0.0.0/::) is reported as 127.0.0.1.

aclose() async

Stop the server, disconnect viewers, and release encoder resources.

Source code in src/pdum/rfb/display.py 
600
601
602
603
604
605
606
async def aclose(self) -> None:
    """Stop the server, disconnect viewers, and release encoder resources."""
    self._close_local()
    if self._server_cm is not None:
        cm, self._server_cm = self._server_cm, None
        self._server = None
        await cm.__aexit__(None, None, None)

events() async

Async-iterate queued events (alternative to :meth:poll_events).

Yields :class:~pdum.rfb.types.InputEvent\ s and, under serve(adaptive=True), :class:~pdum.rfb.types.DownscaleHint\ s. Use this or :meth:poll_events — both drain the same queue.

Source code in src/pdum/rfb/display.py 
443
444
445
446
447
448
449
450
451
452
453
454
455
async def events(self) -> AsyncIterator[InputEvent | DownscaleHint]:
    """Async-iterate queued events (alternative to :meth:`poll_events`).

    Yields :class:`~pdum.rfb.types.InputEvent`\\ s and, under
    ``serve(adaptive=True)``, :class:`~pdum.rfb.types.DownscaleHint`\\ s. Use this or
    :meth:`poll_events` — both drain the same queue.
    """
    while not self._closed:
        if self._events:
            yield self._events.popleft()
            continue
        self._events_signal.clear()
        await self._events_signal.wait()

poll_events()

Drain and return everything queued since the last poll.

The list is ordinarily :class:~pdum.rfb.types.InputEvent\ s (input from all connected viewers). With serve(adaptive=True) it may also carry a :class:~pdum.rfb.types.DownscaleHint — a server-driven resolution suggestion — interleaved in arrival order; tell them apart with isinstance.

Source code in src/pdum/rfb/display.py 
431
432
433
434
435
436
437
438
439
440
441
def poll_events(self) -> list[InputEvent | DownscaleHint]:
    """Drain and return everything queued since the last poll.

    The list is ordinarily :class:`~pdum.rfb.types.InputEvent`\\ s (input from all
    connected viewers). With ``serve(adaptive=True)`` it may also carry a
    :class:`~pdum.rfb.types.DownscaleHint` — a server-driven resolution suggestion —
    interleaved in arrival order; tell them apart with ``isinstance``.
    """
    out = list(self._events)
    self._events.clear()
    return out

publish(frame, *, pixel_ratio=None, color=None)

Make frame the latest frame and wake every connected viewer.

Synchronous and non-blocking. frame may be:

  • a contiguous host uint8 array — (H, W, 3) rgb24 or (H, W, 4) rgba8;
  • a CUDA tensor exposing __cuda_array_interface__ (e.g. CuPy) of shape (H, W, 3|4) — published as a zero-copy cuda frame (for NV12, or other frameworks, build a RawFrame via :func:pdum.rfb.gpu.cuda_frame);
  • an MLX (Apple Metal) array of shape (H, W, 3|4) — published as a metal frame; the VideoToolbox encoder converts RGB(A)→NV12 on the GPU (for pre-converted NV12, use :func:pdum.rfb.metal.metal_frame);
  • a ready :class:~pdum.rfb.types.RawFrame (any memory).

Latest-frame-wins: a viewer that is behind simply skips intermediate frames.

Ownership. By default publish() borrows your buffer — it stores a bare reference and reads the pixels asynchronously, on each viewer's encode worker thread. The borrow window runs from here until every viewer has finished encoding the frame, and it is widest under still_after (the resting frame is re-read ~still_after seconds later for the lossless still). So in borrow mode, publish a fresh buffer each call, or do not mutate a published buffer until it is encoded. Construct the Display (or call :func:~pdum.rfb.serve) with own_frames=True to instead have the server copy each frame into a recycled buffer, after which you may reuse/mutate your own buffer immediately (cpu/cuda only; metal raises).

Parameters:

Name Type Description Default
pixel_ratio float | None

Render-side DPR for this frame (device px per logical px). None keeps a :class:~pdum.rfb.types.RawFrame's own value (else 1.0). See :attr:~pdum.rfb.types.RawFrame.pixel_ratio.

 None
color Any

Color descriptor for this frame — a :class:~pdum.rfb.types.ColorSpace, its dict form, or None (sRGB). The renderer must already produce pixels in the declared space; the library only tags them.

 None
Source code in src/pdum/rfb/display.py 
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
def publish(
    self,
    frame: np.ndarray | RawFrame | Any,
    *,
    pixel_ratio: float | None = None,
    color: Any = None,
) -> None:
    """Make ``frame`` the latest frame and wake every connected viewer.

    Synchronous and non-blocking. ``frame`` may be:

    * a contiguous host ``uint8`` array — ``(H, W, 3)`` ``rgb24`` or
      ``(H, W, 4)`` ``rgba8``;
    * a **CUDA tensor** exposing ``__cuda_array_interface__`` (e.g. CuPy) of
      shape ``(H, W, 3|4)`` — published as a zero-copy ``cuda`` frame (for
      NV12, or other frameworks, build a ``RawFrame`` via
      :func:`pdum.rfb.gpu.cuda_frame`);
    * an **MLX (Apple Metal) array** of shape ``(H, W, 3|4)`` — published as a
      ``metal`` frame; the VideoToolbox encoder converts RGB(A)→NV12 on the GPU
      (for pre-converted NV12, use :func:`pdum.rfb.metal.metal_frame`);
    * a ready :class:`~pdum.rfb.types.RawFrame` (any ``memory``).

    Latest-frame-wins: a viewer that is behind simply skips intermediate frames.

    **Ownership.** By default ``publish()`` *borrows* your buffer — it stores a bare
    reference and reads the pixels **asynchronously**, on each viewer's encode worker
    thread. The borrow window runs from here until every viewer has finished encoding
    the frame, and it is **widest under** ``still_after`` (the resting frame is re-read
    ~``still_after`` seconds later for the lossless still). So in borrow mode, publish a
    *fresh* buffer each call, or do not mutate a published buffer until it is encoded.
    Construct the ``Display`` (or call :func:`~pdum.rfb.serve`) with ``own_frames=True``
    to instead have the server copy each frame into a recycled buffer, after which you
    may reuse/mutate your own buffer immediately (``cpu``/``cuda`` only; ``metal`` raises).

    Parameters
    ----------
    pixel_ratio:
        Render-side DPR for this frame (device px per logical px). ``None`` keeps a
        :class:`~pdum.rfb.types.RawFrame`'s own value (else ``1.0``). See
        :attr:`~pdum.rfb.types.RawFrame.pixel_ratio`.
    color:
        Color descriptor for this frame — a :class:`~pdum.rfb.types.ColorSpace`, its
        ``dict`` form, or ``None`` (sRGB). The renderer must already produce pixels in
        the declared space; the library only tags them.
    """
    if self._closed:
        raise RuntimeError("publish() called on a closed Display")

    frame_pr = frame.pixel_ratio if isinstance(frame, RawFrame) else 1.0
    frame_color = frame.color if isinstance(frame, RawFrame) else None
    resolved_pr = float(frame_pr if pixel_ratio is None else pixel_ratio)
    resolved_color = frame_color if color is None else _color_to_dict(color)

    if isinstance(frame, RawFrame):
        data, width, height = frame.data, frame.width, frame.height
        pixel_format, memory = frame.pixel_format, frame.memory
    elif isinstance(frame, np.ndarray):
        if frame.ndim != 3 or frame.shape[2] not in (3, 4):
            raise ValueError(f"unsupported frame shape {frame.shape!r}; expected (H, W, 3) or (H, W, 4)")
        height, width = int(frame.shape[0]), int(frame.shape[1])
        pixel_format = "rgb24" if frame.shape[2] == 3 else "rgba8"
        data, memory = frame, "cpu"
    elif _is_cuda_tensor(frame):
        shape = getattr(frame, "shape", None)
        if shape is None or len(shape) != 3 or shape[2] not in (3, 4):
            raise ValueError("CUDA publish expects an (H, W, 3|4) tensor; use pdum.rfb.gpu.cuda_frame() for NV12")
        height, width = int(shape[0]), int(shape[1])
        pixel_format = "rgb24" if shape[2] == 3 else "rgba8"
        data, memory = frame, "cuda"
    elif _is_metal_tensor(frame):
        shape = getattr(frame, "shape", None)
        if shape is None or len(shape) != 3 or shape[2] not in (3, 4):
            raise ValueError(
                "Metal publish expects an (H, W, 3|4) MLX array; use pdum.rfb.metal.metal_frame() for NV12"
            )
        height, width = int(shape[0]), int(shape[1])
        pixel_format = "rgb24" if shape[2] == 3 else "rgba8"
        data, memory = frame, "metal"
    else:
        raise TypeError("publish() expects a numpy.ndarray, a CUDA/Metal tensor, or a RawFrame")

    if memory == "metal":
        # Materialize the lazy MLX render on *this* (loop) thread: MLX binds a lazy graph to
        # its origin thread's stream, so the session's encode worker thread cannot evaluate a
        # frame built here. The GPU NV12 conversion still runs on the worker. See metal.materialize.
        from .metal import materialize

        materialize(data)

    if self._own_frames:
        # Server-owned mode: copy into a recycled buffer on this (loop) thread so the caller
        # may reuse/mutate its own buffer immediately. Severs the async-read aliasing entirely.
        data = self._own_copy(data, memory)

    timestamp_us = int((self._clock() - self._start) * 1_000_000)
    # seq is a placeholder; each feed stamps its own per-client sequence.
    self._latest = RawFrame(
        seq=0,
        width=width,
        height=height,
        timestamp_us=timestamp_us,
        pixel_format=pixel_format,  # type: ignore[arg-type]
        memory=memory,  # type: ignore[arg-type]
        data=data,
        pixel_ratio=resolved_pr,
        color=resolved_color,
    )
    self.width, self.height = width, height
    self._version += 1
    for feed in self._feeds:
        feed._wake()
    for tap in self._taps:
        tap._wake()

record(path, *, fps=None, bitrate=None, crf=None, preset='veryfast')

Record the published-frame stream to a real MP4 file (server-side tap).

Returns a started :class:~pdum.rfb.recording.Recording — an async context manager / handle with :meth:~pdum.rfb.recording.Recording.stop. It taps the same latest-frame stream the viewer sessions pull from, so it is independent of any connected browser and works fully headless (demos, CI artifacts, golden-frame capture). Frames are H.264-encoded (libx264) and muxed to AVCC-in-MP4 via a separate PyAV path — this is the one place AVCC/MP4 is correct; it never touches the Annex-B WebSocket payload.

The real, monotonic per-frame timestamps are honored, so a variable-cadence publisher (sparse still_after scenes included) records with correct frame timing. Call it on the event-loop thread; it starts a background encode task.

::

async with display.record("out.mp4"):
    for _ in range(300):
        display.publish(render()); await asyncio.sleep(1 / 30)
# file finalized on exit — or use rec = display.record(...); await rec.stop()

Parameters:

Name Type Description Default
path str | Path

Output .mp4 file path.

 required
fps int | None

Advisory frame rate for the encoder's rate control (default: the display's fps). The actual timing comes from the frames' real timestamps.

 None
bitrate int | None

Target bitrate in bits/s (average). Mutually exclusive with crf; if neither is given, a sensible constant-quality crf is used.

 None
crf int | None

Constant Rate Factor (0–51, lower = better) for constant-quality encoding.

 None
preset str

libx264 speed/quality preset (default "veryfast").

 'veryfast'

Raises:

Type Description
RuntimeError

If PyAV (the [h264] extra) is not installed.
Source code in src/pdum/rfb/display.py 
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
def record(
    self,
    path: str | Path,
    *,
    fps: int | None = None,
    bitrate: int | None = None,
    crf: int | None = None,
    preset: str = "veryfast",
) -> Recording:
    """Record the published-frame stream to a real **MP4** file (server-side tap).

    Returns a started :class:`~pdum.rfb.recording.Recording` — an async context
    manager / handle with :meth:`~pdum.rfb.recording.Recording.stop`. It taps the
    same latest-frame stream the viewer sessions pull from, so it is **independent
    of any connected browser** and works fully headless (demos, CI artifacts,
    golden-frame capture). Frames are H.264-encoded (libx264) and muxed to
    **AVCC-in-MP4** via a **separate** PyAV path — this is the one place AVCC/MP4 is
    correct; it never touches the Annex-B WebSocket payload.

    The real, monotonic per-frame timestamps are honored, so a variable-cadence
    publisher (sparse ``still_after`` scenes included) records with correct frame
    timing. Call it on the event-loop thread; it starts a background encode task.

    ::

        async with display.record("out.mp4"):
            for _ in range(300):
                display.publish(render()); await asyncio.sleep(1 / 30)
        # file finalized on exit — or use rec = display.record(...); await rec.stop()

    Parameters
    ----------
    path:
        Output ``.mp4`` file path.
    fps:
        Advisory frame rate for the encoder's rate control (default: the display's
        ``fps``). The *actual* timing comes from the frames' real timestamps.
    bitrate:
        Target bitrate in bits/s (average). Mutually exclusive with ``crf``; if
        neither is given, a sensible constant-quality ``crf`` is used.
    crf:
        Constant Rate Factor (0–51, lower = better) for constant-quality encoding.
    preset:
        libx264 speed/quality preset (default ``"veryfast"``).

    Raises
    ------
    RuntimeError
        If PyAV (the ``[h264]`` extra) is not installed.
    """
    from .recording import Recording

    rec = Recording(self, path, fps=fps or self.fps, bitrate=bitrate, crf=crf, preset=preset)
    rec._start_task()
    return rec

widget(*, batteries=True, base_path=None, host=None, **chrome)

Return an anywidget viewer bound to this stream (needs the [anywidget] extra).

In a notebook: display.widget() renders the batteries viewer; display.widget(batteries=False) the bare canvas. One widget = one Web Worker + one WebSocket; the server multiplexes many streams on one port. For remote/HTTPS notebooks, mount pdum.rfb.asgi.rfb_hub_endpoint same-origin and pass base_path= (the widget then uses a same-origin wss:// URL). Extra keyword args (e.g. show_toolbar=False) become widget traits.

Raises if the server is not bound yet.

Source code in src/pdum/rfb/display.py 
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
def widget(
    self,
    *,
    batteries: bool = True,
    base_path: str | None = None,
    host: str | None = None,
    **chrome: Any,
) -> Any:
    """Return an anywidget viewer bound to this stream (needs the ``[anywidget]`` extra).

    In a notebook: ``display.widget()`` renders the batteries viewer;
    ``display.widget(batteries=False)`` the bare canvas. One widget = one Web Worker +
    one WebSocket; the server multiplexes many streams on one port. For remote/HTTPS
    notebooks, mount ``pdum.rfb.asgi.rfb_hub_endpoint`` same-origin and pass
    ``base_path=`` (the widget then uses a same-origin ``wss://`` URL). Extra keyword
    args (e.g. ``show_toolbar=False``) become widget traits.

    Raises if the server is not bound yet.
    """
    from .notebook import RfbCanvas, RfbViewer

    if self.port is None:
        raise RuntimeError("Display is not serving yet; call await rfb.serve(...) first")
    resolved_host = host if host is not None else (getattr(self._owner_server, "host", None) or "127.0.0.1")
    if resolved_host in ("0.0.0.0", "", "::"):
        # Let the browser use the page's own hostname (correct for remote notebooks).
        resolved_host = "auto"
    cls = RfbViewer if batteries else RfbCanvas
    kwargs: dict[str, Any] = {"port": self.port, "stream": self._stream_name, "host": resolved_host, **chrome}
    if base_path is not None:
        kwargs["base_path"] = base_path
    return cls(**kwargs)

encoders

Encoder backends for the remote framebuffer.

ImageEncoder

Encode CPU RGB/RGBA frames to JPEG, PNG or WebP.

Source code in src/pdum/rfb/encoders/image.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
class ImageEncoder:
    """Encode CPU RGB/RGBA frames to JPEG, PNG or WebP."""

    def __init__(self, *, mode: ImageMode = "jpeg", quality: int = 80) -> None:
        self.mode: ImageMode = mode
        self.quality = quality

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        if frame.memory == "metal":  # a published MLX frame: download to host rgb24
            from ..metal import to_host_frame

            frame = to_host_frame(frame)
        if frame.memory != "cpu":
            raise TypeError("ImageEncoder expects CPU frames")

        arr = frame.data
        if not isinstance(arr, np.ndarray):
            raise TypeError("Expected numpy.ndarray")

        if frame.pixel_format == "rgb24":
            img = Image.fromarray(arr, "RGB")
        elif frame.pixel_format == "rgba8":
            img = Image.fromarray(arr, "RGBA")
        else:
            raise ValueError(f"Unsupported pixel format for image encoder: {frame.pixel_format}")

        out = BytesIO()
        if self.mode == "jpeg":
            if img.mode == "RGBA":  # JPEG cannot store alpha.
                img = img.convert("RGB")
            img.save(out, format="JPEG", quality=self.quality, optimize=False)
            mime = CAP_JPEG
        elif self.mode == "png":
            img.save(out, format="PNG")
            mime = CAP_PNG
        elif self.mode == "webp":
            img.save(out, format="WEBP", quality=self.quality)
            mime = CAP_WEBP
        else:  # pragma: no cover - guarded by the Literal type
            raise ValueError(self.mode)

        return [
            EncodedPayload(
                seq=frame.seq,
                kind="image",
                timestamp_us=frame.timestamp_us,
                width=frame.width,
                height=frame.height,
                mime=mime,
                payload=out.getvalue(),
                keyframe=True,
            )
        ]

    def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
        """Encode ``frame`` losslessly as **PNG** — the "still after settle" upgrade.

        Independent of the streaming ``mode``: once a scene settles, the resting
        frame is re-sent pixel-exact so it is crisp even when the live stream is
        lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it
        in with no client-side changes.
        """
        return ImageEncoder(mode="png").encode(frame)

    def flush(self) -> list[EncodedPayload]:
        return []

    def close(self) -> None:
        pass

encode_still(frame)

Encode frame losslessly as PNG — the "still after settle" upgrade.

Independent of the streaming mode: once a scene settles, the resting frame is re-sent pixel-exact so it is crisp even when the live stream is lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it in with no client-side changes.

Source code in src/pdum/rfb/encoders/image.py 
76
77
78
79
80
81
82
83
84
def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
    """Encode ``frame`` losslessly as **PNG** — the "still after settle" upgrade.

    Independent of the streaming ``mode``: once a scene settles, the resting
    frame is re-sent pixel-exact so it is crisp even when the live stream is
    lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it
    in with no client-side changes.
    """
    return ImageEncoder(mode="png").encode(frame)

available_video_encoders()

Return the names of registered video encoders.

Source code in src/pdum/rfb/encoders/base.py 
29
30
31
def available_video_encoders() -> list[str]:
    """Return the names of registered video encoders."""
    return sorted(_VIDEO_ENCODERS)

build_encoder(selection, *, width, height, fps=30, bitrate=12000000, video_encoder='h264_cpu', pipeline_depth=0, color=None)

Build the encoder backend described by selection.

Parameters:

Name Type Description Default
selection BackendSelection

The result of :func:pdum.rfb.protocol.select_transport.

 required
width int

Encoder configuration (ignored by the image encoder except where noted).

 required
height int

Encoder configuration (ignored by the image encoder except where noted).

 required
fps int

Encoder configuration (ignored by the image encoder except where noted).

 required
bitrate int

Encoder configuration (ignored by the image encoder except where noted).

 required
video_encoder str

Which registered video encoder to use for the H.264 transport.

 'h264_cpu'
pipeline_depth int

Encoder pipeline depth. 0 (default) is synchronous 1-in-1-out (lowest latency, seq attribution trivially correct). > 0 opts into the token-based pipelined path on backends that implement it (NVENC; on VideoToolbox it is correct but not faster). See :doc:pipelined_encode.

 0
color dict | None

Optional stream color descriptor (dict form of a :class:~pdum.rfb.types.ColorSpace). The PyAV H.264 backends tag the bitstream VUI with it; the GPU-SDK / VideoToolbox backends currently ignore it (native follow-up).

 None
Source code in src/pdum/rfb/encoders/base.py 
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def build_encoder(
    selection: BackendSelection,
    *,
    width: int,
    height: int,
    fps: int = 30,
    bitrate: int = 12_000_000,
    video_encoder: str = "h264_cpu",
    pipeline_depth: int = 0,
    color: dict | None = None,
) -> EncoderBackend:
    """Build the encoder backend described by ``selection``.

    Parameters
    ----------
    selection:
        The result of :func:`pdum.rfb.protocol.select_transport`.
    width, height, fps, bitrate:
        Encoder configuration (ignored by the image encoder except where noted).
    video_encoder:
        Which registered video encoder to use for the H.264 transport.
    pipeline_depth:
        Encoder pipeline depth. ``0`` (default) is synchronous 1-in-1-out (lowest latency,
        seq attribution trivially correct). ``> 0`` opts into the token-based pipelined path
        on backends that implement it (NVENC; on VideoToolbox it is correct but not faster).
        See :doc:`pipelined_encode`.
    color:
        Optional stream color descriptor (``dict`` form of a
        :class:`~pdum.rfb.types.ColorSpace`). The PyAV H.264 backends tag the bitstream VUI
        with it; the GPU-SDK / VideoToolbox backends currently ignore it (native follow-up).
    """
    if selection.transport == "image":
        return ImageEncoder(mode=selection.image_mode or "jpeg")

    if selection.transport == "h264":
        try:
            factory = _VIDEO_ENCODERS[video_encoder]
        except KeyError as exc:
            raise ValueError(
                f"unknown video encoder {video_encoder!r}; available: {available_video_encoders()}"
            ) from exc
        return factory(
            width=width,
            height=height,
            fps=fps,
            bitrate=bitrate,
            codec_string=selection.codec,
            pipeline_depth=pipeline_depth,
            color=color,
        )

    raise ValueError(f"unsupported transport: {selection.transport!r}")

register_video_encoder(name, factory)

Register a video :class:EncoderBackend factory under name.

Source code in src/pdum/rfb/encoders/base.py 
24
25
26
def register_video_encoder(name: str, factory: EncoderFactory) -> None:
    """Register a video :class:`EncoderBackend` factory under ``name``."""
    _VIDEO_ENCODERS[name] = factory

base

Encoder registry and the build_encoder factory.

The registry is the extension seam for additional video encoders. The CPU H.264 (PyAV/libx264) backend registers itself lazily so importing this module never imports PyAV. The NVENC backends register themselves the same way and :func:pdum.rfb.protocol.select_transport flips has_nvenc to prefer them.

available_video_encoders()

Return the names of registered video encoders.

Source code in src/pdum/rfb/encoders/base.py 
29
30
31
def available_video_encoders() -> list[str]:
    """Return the names of registered video encoders."""
    return sorted(_VIDEO_ENCODERS)

build_encoder(selection, *, width, height, fps=30, bitrate=12000000, video_encoder='h264_cpu', pipeline_depth=0, color=None)

Build the encoder backend described by selection.

Parameters:

Name Type Description Default
selection BackendSelection

The result of :func:pdum.rfb.protocol.select_transport.

 required
width int

Encoder configuration (ignored by the image encoder except where noted).

 required
height int

Encoder configuration (ignored by the image encoder except where noted).

 required
fps int

Encoder configuration (ignored by the image encoder except where noted).

 required
bitrate int

Encoder configuration (ignored by the image encoder except where noted).

 required
video_encoder str

Which registered video encoder to use for the H.264 transport.

 'h264_cpu'
pipeline_depth int

Encoder pipeline depth. 0 (default) is synchronous 1-in-1-out (lowest latency, seq attribution trivially correct). > 0 opts into the token-based pipelined path on backends that implement it (NVENC; on VideoToolbox it is correct but not faster). See :doc:pipelined_encode.

 0
color dict | None

Optional stream color descriptor (dict form of a :class:~pdum.rfb.types.ColorSpace). The PyAV H.264 backends tag the bitstream VUI with it; the GPU-SDK / VideoToolbox backends currently ignore it (native follow-up).

 None
Source code in src/pdum/rfb/encoders/base.py 
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
def build_encoder(
    selection: BackendSelection,
    *,
    width: int,
    height: int,
    fps: int = 30,
    bitrate: int = 12_000_000,
    video_encoder: str = "h264_cpu",
    pipeline_depth: int = 0,
    color: dict | None = None,
) -> EncoderBackend:
    """Build the encoder backend described by ``selection``.

    Parameters
    ----------
    selection:
        The result of :func:`pdum.rfb.protocol.select_transport`.
    width, height, fps, bitrate:
        Encoder configuration (ignored by the image encoder except where noted).
    video_encoder:
        Which registered video encoder to use for the H.264 transport.
    pipeline_depth:
        Encoder pipeline depth. ``0`` (default) is synchronous 1-in-1-out (lowest latency,
        seq attribution trivially correct). ``> 0`` opts into the token-based pipelined path
        on backends that implement it (NVENC; on VideoToolbox it is correct but not faster).
        See :doc:`pipelined_encode`.
    color:
        Optional stream color descriptor (``dict`` form of a
        :class:`~pdum.rfb.types.ColorSpace`). The PyAV H.264 backends tag the bitstream VUI
        with it; the GPU-SDK / VideoToolbox backends currently ignore it (native follow-up).
    """
    if selection.transport == "image":
        return ImageEncoder(mode=selection.image_mode or "jpeg")

    if selection.transport == "h264":
        try:
            factory = _VIDEO_ENCODERS[video_encoder]
        except KeyError as exc:
            raise ValueError(
                f"unknown video encoder {video_encoder!r}; available: {available_video_encoders()}"
            ) from exc
        return factory(
            width=width,
            height=height,
            fps=fps,
            bitrate=bitrate,
            codec_string=selection.codec,
            pipeline_depth=pipeline_depth,
            color=color,
        )

    raise ValueError(f"unsupported transport: {selection.transport!r}")

register_video_encoder(name, factory)

Register a video :class:EncoderBackend factory under name.

Source code in src/pdum/rfb/encoders/base.py 
24
25
26
def register_video_encoder(name: str, factory: EncoderFactory) -> None:
    """Register a video :class:`EncoderBackend` factory under ``name``."""
    _VIDEO_ENCODERS[name] = factory

h264_cpu

CPU H.264 encoder via PyAV / libx264 (guide section 6).

Produces Annex B access units (in-band SPS/PPS on key frames) suitable for the browser's WebCodecs VideoDecoder in Annex B mode. Configured for low latency: ultrafast/zerolatency, no B-frames, periodic IDR.

Several gaps in the guide's sketch are fixed here:

  • forced keyframes are real IDRs (forced-idr=1 + per-frame pict_type=I);
  • RGB is explicitly reformatted to yuv420p (PyAV does not auto-convert);
  • annexb=1 / repeat-headers=1 keep parameter sets in-band.

PyAV is imported lazily so this module can be imported (e.g. for :func:h264_available) even where av is installed only as the optional h264 extra.

H264CpuEncoder

Encode CPU rgb24 frames to H.264 Annex B access units.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
class H264CpuEncoder:
    """Encode CPU ``rgb24`` frames to H.264 Annex B access units."""

    #: Recorded in each payload's ``metadata["encoder"]`` so the wire/headers
    #: identify which backend produced the bitstream. Subclasses override it.
    encoder_label = "h264-cpu"

    def __init__(
        self,
        *,
        width: int,
        height: int,
        fps: int = 30,
        bitrate: int = 12_000_000,
        codec_string: str | None = None,
        color: dict | None = None,
    ) -> None:
        self.width = width
        self.height = height
        self.fps = fps
        self.bitrate = bitrate
        self.codec_string = codec_string or DEFAULT_H264_CODEC
        self.color = color
        self.frame_index = 0
        self._duration_us = int(1_000_000 / fps)
        self.ctx = self._make_context()
        self._apply_color_vui(self.ctx)

    def _make_context(self):
        """Build the libx264 :class:`av.CodecContext` (overridden by NVENC)."""
        import av

        ctx = av.CodecContext.create("libx264", "w")
        ctx.width = self.width
        ctx.height = self.height
        ctx.pix_fmt = "yuv420p"
        ctx.time_base = Fraction(1, self.fps)
        ctx.framerate = Fraction(self.fps, 1)
        ctx.bit_rate = self.bitrate
        # Low latency: ultrafast, zerolatency, no B-frames, periodic in-band IDR.
        ctx.options = {
            "preset": "ultrafast",
            "tune": "zerolatency",
            "profile": "baseline",
            "forced-idr": "1",
            "x264-params": (f"keyint={self.fps}:min-keyint={self.fps}:scenecut=0:bframes=0:annexb=1:repeat-headers=1"),
        }
        return ctx

    def _apply_color_vui(self, ctx) -> None:
        """Signal color primaries/transfer/matrix/range on the codec context (→ H.264 VUI),
        shared by the libx264 and NVENC contexts. No-op for the default sRGB stream."""
        codes = h264_color_vui(self.color)
        if codes is None:
            return
        primaries, transfer, matrix, color_range = codes
        ctx.color_primaries = primaries
        ctx.color_trc = transfer
        ctx.colorspace = matrix
        ctx.color_range = color_range

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        import av

        if frame.memory == "metal":  # a published MLX frame: download to host rgb24
            from ..metal import to_host_frame

            frame = to_host_frame(frame)
        if frame.memory != "cpu" or frame.pixel_format != "rgb24":
            raise TypeError("H264CpuEncoder expects CPU rgb24 frames")
        arr = frame.data
        if not isinstance(arr, np.ndarray):
            raise TypeError("Expected numpy.ndarray")

        vf = av.VideoFrame.from_ndarray(np.ascontiguousarray(arr), format="rgb24")
        vf = vf.reformat(format="yuv420p")
        vf.pts = self.frame_index
        vf.time_base = Fraction(1, self.fps)
        if force_keyframe:
            vf.pict_type = av.video.frame.PictureType.I
        self.frame_index += 1

        return [self._payload(frame.seq, frame.timestamp_us, pkt) for pkt in self._drain(vf)]

    def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
        """A settled-scene still for the video path: a forced **IDR** of the frame.

        True lossless H.264 isn't practical over WebCodecs, so the still here is a
        clean, self-contained intra (IDR) of the resting frame — it refreshes the
        image and lets a client that dropped deltas during a flurry jump straight
        to the latest. Re-encoding advances ``frame_index`` so the PTS stays
        monotonic. For a pixel-exact settled image, use the image transport.
        """
        return self.encode(frame, force_keyframe=True)

    def flush(self) -> list[EncodedPayload]:
        return [self._payload(-1, 0, pkt) for pkt in self._drain(None)]

    def close(self) -> None:
        try:
            self.flush()
        except Exception:  # pragma: no cover - encoder may already be closed
            pass

    # --- helpers ------------------------------------------------------------

    def _drain(self, vf):
        for packet in self.ctx.encode(vf):
            data = bytes(packet)
            if data:
                yield packet, data

    def _payload(self, seq: int, timestamp_us: int, pkt) -> EncodedPayload:
        packet, data = pkt
        return EncodedPayload(
            seq=seq,
            kind="video",
            timestamp_us=timestamp_us,
            width=self.width,
            height=self.height,
            payload=data,
            codec=self.codec_string,
            keyframe=bool(packet.is_keyframe),
            duration_us=self._duration_us,
            metadata={"bitstream": "annexb", "encoder": self.encoder_label},
        )
encode_still(frame)

A settled-scene still for the video path: a forced IDR of the frame.

True lossless H.264 isn't practical over WebCodecs, so the still here is a clean, self-contained intra (IDR) of the resting frame — it refreshes the image and lets a client that dropped deltas during a flurry jump straight to the latest. Re-encoding advances frame_index so the PTS stays monotonic. For a pixel-exact settled image, use the image transport.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
154
155
156
157
158
159
160
161
162
163
def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
    """A settled-scene still for the video path: a forced **IDR** of the frame.

    True lossless H.264 isn't practical over WebCodecs, so the still here is a
    clean, self-contained intra (IDR) of the resting frame — it refreshes the
    image and lets a client that dropped deltas during a flurry jump straight
    to the latest. Re-encoding advances ``frame_index`` so the PTS stays
    monotonic. For a pixel-exact settled image, use the image transport.
    """
    return self.encode(frame, force_keyframe=True)

h264_available()

True if PyAV is importable.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
29
30
31
def h264_available() -> bool:
    """True if PyAV is importable."""
    return importlib.util.find_spec("av") is not None

h264_color_vui(color)

Map a color descriptor to (primaries, transfer, matrix, range) ffmpeg codes.

Returns None for the default (color is None) so no VUI is written and existing sRGB streams are bitstream-unchanged.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
45
46
47
48
49
50
51
52
53
54
55
def h264_color_vui(color: dict | None) -> tuple[int, int, int, int] | None:
    """Map a color descriptor to ``(primaries, transfer, matrix, range)`` ffmpeg codes.

    Returns ``None`` for the default (``color is None``) so no VUI is written and existing
    sRGB streams are bitstream-unchanged.
    """
    if not color:
        return None
    primaries = _AV_PRIMARIES.get(color.get("primaries", "bt709"), 1)
    transfer = _AV_TRANSFER.get(color.get("transfer", "srgb"), 13)
    return (primaries, transfer, _AV_MATRIX_BT601, _AV_RANGE_LIMITED)

h264_cpu_available()

True if PyAV is importable and exposes the libx264 encoder.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
58
59
60
61
62
63
64
65
66
67
def h264_cpu_available() -> bool:
    """True if PyAV is importable and exposes the libx264 encoder."""
    if not h264_available():
        return False
    try:
        import av

        return "libx264" in av.codecs_available
    except Exception:  # pragma: no cover - defensive
        return False

self_test(width=64, height=64, frames=8)

Encode a few synthetic frames and decode them back to prove validity.

Returns True if the produced Annex B bitstream decodes to a plausible number of frames at the expected resolution. Doubles as a runtime check that libx264 is actually usable.

Source code in src/pdum/rfb/encoders/h264_cpu.py 
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
def self_test(width: int = 64, height: int = 64, frames: int = 8) -> bool:
    """Encode a few synthetic frames and decode them back to prove validity.

    Returns ``True`` if the produced Annex B bitstream decodes to a plausible
    number of frames at the expected resolution. Doubles as a runtime check
    that libx264 is actually usable.
    """
    if not h264_cpu_available():
        return False

    from ..testing import decode_annexb

    enc = H264CpuEncoder(width=width, height=height, fps=int(frames))
    chunks: list[bytes] = []
    for seq in range(frames):
        arr = np.full((height, width, 3), (seq * 7) % 256, dtype=np.uint8)
        arr[:, : width // 2] = ((seq * 11) % 256, (seq * 5) % 256, 64)
        for payload in enc.encode(
            RawFrame(seq, width, height, seq * 1000, "rgb24", "cpu", arr),
            force_keyframe=(seq == 0),
        ):
            chunks.append(payload.payload)
    for payload in enc.flush():
        chunks.append(payload.payload)
    enc.close()

    decoded = decode_annexb(b"".join(chunks))
    if not decoded:
        return False
    return all(f.width == width and f.height == height for f in decoded)

image

Image encoder: JPEG / PNG / WebP via Pillow.

The simplest backend (guide section 5). Every image is an independent frame, so every payload is a keyframe. Ideal for snapshots, debug views, low/medium frame rates, and mostly static plots.

ImageEncoder

Encode CPU RGB/RGBA frames to JPEG, PNG or WebP.

Source code in src/pdum/rfb/encoders/image.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
class ImageEncoder:
    """Encode CPU RGB/RGBA frames to JPEG, PNG or WebP."""

    def __init__(self, *, mode: ImageMode = "jpeg", quality: int = 80) -> None:
        self.mode: ImageMode = mode
        self.quality = quality

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        if frame.memory == "metal":  # a published MLX frame: download to host rgb24
            from ..metal import to_host_frame

            frame = to_host_frame(frame)
        if frame.memory != "cpu":
            raise TypeError("ImageEncoder expects CPU frames")

        arr = frame.data
        if not isinstance(arr, np.ndarray):
            raise TypeError("Expected numpy.ndarray")

        if frame.pixel_format == "rgb24":
            img = Image.fromarray(arr, "RGB")
        elif frame.pixel_format == "rgba8":
            img = Image.fromarray(arr, "RGBA")
        else:
            raise ValueError(f"Unsupported pixel format for image encoder: {frame.pixel_format}")

        out = BytesIO()
        if self.mode == "jpeg":
            if img.mode == "RGBA":  # JPEG cannot store alpha.
                img = img.convert("RGB")
            img.save(out, format="JPEG", quality=self.quality, optimize=False)
            mime = CAP_JPEG
        elif self.mode == "png":
            img.save(out, format="PNG")
            mime = CAP_PNG
        elif self.mode == "webp":
            img.save(out, format="WEBP", quality=self.quality)
            mime = CAP_WEBP
        else:  # pragma: no cover - guarded by the Literal type
            raise ValueError(self.mode)

        return [
            EncodedPayload(
                seq=frame.seq,
                kind="image",
                timestamp_us=frame.timestamp_us,
                width=frame.width,
                height=frame.height,
                mime=mime,
                payload=out.getvalue(),
                keyframe=True,
            )
        ]

    def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
        """Encode ``frame`` losslessly as **PNG** — the "still after settle" upgrade.

        Independent of the streaming ``mode``: once a scene settles, the resting
        frame is re-sent pixel-exact so it is crisp even when the live stream is
        lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it
        in with no client-side changes.
        """
        return ImageEncoder(mode="png").encode(frame)

    def flush(self) -> list[EncodedPayload]:
        return []

    def close(self) -> None:
        pass
encode_still(frame)

Encode frame losslessly as PNG — the "still after settle" upgrade.

Independent of the streaming mode: once a scene settles, the resting frame is re-sent pixel-exact so it is crisp even when the live stream is lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it in with no client-side changes.

Source code in src/pdum/rfb/encoders/image.py 
76
77
78
79
80
81
82
83
84
def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
    """Encode ``frame`` losslessly as **PNG** — the "still after settle" upgrade.

    Independent of the streaming ``mode``: once a scene settles, the resting
    frame is re-sent pixel-exact so it is crisp even when the live stream is
    lossy JPEG/WebP. Every image is already a keyframe, so the browser swaps it
    in with no client-side changes.
    """
    return ImageEncoder(mode="png").encode(frame)

nvenc_cpu

Hardware H.264 encoder via NVIDIA NVENC (the roadmap's GPU backend).

This rides on PyAV's bundled ffmpeg rather than NVIDIA's PyNvVideoCodec: the av wheel ships an ffmpeg built with --enable-nvenc, exposing the h264_nvenc encoder, which dlopen\s libnvidia-encode at runtime. So this backend needs no Python dependency beyond av (already the h264 / nvenc extra) — only a host NVIDIA driver + an NVENC-capable GPU, which pip cannot install.

Like :class:~pdum.rfb.encoders.h264_cpu.H264CpuEncoder it emits Annex B access units (in-band SPS/PPS on key frames) configured for low latency (preset=p4/tune=ll, no B-frames, ~1 s forced-IDR cadence). It subclasses the libx264 encoder and only swaps the underlying av.CodecContext, so the frame conversion, forced-keyframe handling, and payload packing are shared.

Availability is gated by :func:nvenc_cpu_available, which checks the OS, that PyAV exposes h264_nvenc, and that the encoder actually opens on the GPU.

NvencCpuEncoder

Bases: H264CpuEncoder

Encode CPU rgb24 frames to H.264 Annex B on the GPU via NVENC.

Drop-in for :class:H264CpuEncoder (same constructor and EncoderBackend interface); only the underlying codec context differs. Frames narrower than :data:NVENC_MIN_WIDTH are rejected because NVENC cannot open below it.

Source code in src/pdum/rfb/encoders/nvenc_cpu.py 
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
class NvencCpuEncoder(H264CpuEncoder):
    """Encode CPU ``rgb24`` frames to H.264 Annex B on the GPU via NVENC.

    Drop-in for :class:`H264CpuEncoder` (same constructor and ``EncoderBackend``
    interface); only the underlying codec context differs. Frames narrower than
    :data:`NVENC_MIN_WIDTH` are rejected because NVENC cannot open below it.
    """

    encoder_label = "nvenc-cpu"

    def __init__(
        self,
        *,
        width: int,
        height: int,
        fps: int = 30,
        bitrate: int = 12_000_000,
        codec_string: str | None = None,
        color: dict | None = None,
    ) -> None:
        if width < NVENC_MIN_WIDTH:
            raise ValueError(
                f"NVENC requires width >= {NVENC_MIN_WIDTH}; got {width}. "
                "Use a larger framebuffer or fall back to the libx264 encoder."
            )
        super().__init__(
            width=width,
            height=height,
            fps=fps,
            bitrate=bitrate,
            codec_string=codec_string or DEFAULT_H264_CODEC,
            color=color,
        )

    def _make_context(self):
        import av

        ctx = av.CodecContext.create(_NVENC_CODEC, "w")
        ctx.width = self.width
        ctx.height = self.height
        ctx.pix_fmt = "yuv420p"
        ctx.time_base = Fraction(1, self.fps)
        ctx.framerate = Fraction(self.fps, 1)
        ctx.bit_rate = self.bitrate
        # Low latency: fast preset, low-latency tune, no B-frames, immediate
        # output, ~1 s forced-IDR cadence. ``rc=vbr`` treats ``bit_rate`` as a
        # target/ceiling and lets the stream **undershoot** on static frames —
        # critical for sparse scientific scenes (CBR would pad to the full bitrate
        # even when nothing changed, ~60x more bytes for identical quality, and
        # mirrors how the libx264 ABR path already behaves). ``profile=baseline``
        # makes the SPS advertise profile_idc 66 to match the negotiated
        # ``avc1.42E01F`` codec string (NVENC otherwise defaults to main/high).
        # NVENC emits Annex B with in-band SPS/PPS on every IDR for a raw stream.
        ctx.options = {
            "preset": "p4",
            "tune": "ll",
            "rc": "vbr",
            "profile": "baseline",
            "bf": "0",
            "delay": "0",
            "forced-idr": "1",
            "g": str(self.fps),
        }
        return ctx

nvenc_codec_available()

True if PyAV is importable and lists the h264_nvenc encoder.

This is a cheap, side-effect-free check (it does not touch the GPU). It is necessary but not sufficient — the encoder still has to open on a real device, which :func:nvenc_cpu_available verifies.

Source code in src/pdum/rfb/encoders/nvenc_cpu.py 
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def nvenc_codec_available() -> bool:
    """True if PyAV is importable and lists the ``h264_nvenc`` encoder.

    This is a cheap, side-effect-free check (it does not touch the GPU). It is
    necessary but not sufficient — the encoder still has to *open* on a real
    device, which :func:`nvenc_cpu_available` verifies.
    """
    if sys.platform not in _NVENC_PLATFORMS:
        return False
    if not h264_available():
        return False
    try:
        import av

        return _NVENC_CODEC in av.codecs_available
    except Exception:  # pragma: no cover - defensive
        return False

nvenc_cpu_available()

True if a usable NVENC H.264 encoder is present (cached).

Guards, in order: the OS must be one where NVENC exists (not macOS); PyAV must expose h264_nvenc; and the encoder must actually open and encode a frame on the GPU (which requires an NVENC-capable device and a working driver). The result is cached for the lifetime of the process.

Source code in src/pdum/rfb/encoders/nvenc_cpu.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
def nvenc_cpu_available() -> bool:
    """True if a usable NVENC H.264 encoder is present (cached).

    Guards, in order: the OS must be one where NVENC exists (not macOS); PyAV
    must expose ``h264_nvenc``; and the encoder must actually open and encode a
    frame on the GPU (which requires an NVENC-capable device and a working
    driver). The result is cached for the lifetime of the process.
    """
    global _nvenc_ok
    if _nvenc_ok is not None:
        return _nvenc_ok
    if not nvenc_codec_available():
        _nvenc_ok = False
        return _nvenc_ok
    _nvenc_ok = _probe_open()
    return _nvenc_ok

self_test(width=256, height=256, frames=8)

Encode a few synthetic frames via NVENC and decode them back with PyAV.

Returns True if the produced Annex B bitstream decodes to a plausible number of frames at the expected resolution. Doubles as a runtime check that NVENC is actually usable end-to-end.

Source code in src/pdum/rfb/encoders/nvenc_cpu.py 
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
def self_test(width: int = 256, height: int = 256, frames: int = 8) -> bool:
    """Encode a few synthetic frames via NVENC and decode them back with PyAV.

    Returns ``True`` if the produced Annex B bitstream decodes to a plausible
    number of frames at the expected resolution. Doubles as a runtime check that
    NVENC is actually usable end-to-end.
    """
    if not nvenc_cpu_available():
        return False

    import numpy as np

    from ..testing import decode_annexb
    from ..types import RawFrame

    width = max(width, NVENC_MIN_WIDTH)
    enc = NvencCpuEncoder(width=width, height=height, fps=int(frames))
    chunks: list[bytes] = []
    for seq in range(frames):
        arr = np.full((height, width, 3), (seq * 7) % 256, dtype=np.uint8)
        arr[:, : width // 2] = ((seq * 11) % 256, (seq * 5) % 256, 64)
        for payload in enc.encode(
            RawFrame(seq, width, height, seq * 1000, "rgb24", "cpu", arr),
            force_keyframe=(seq == 0),
        ):
            chunks.append(payload.payload)
    for payload in enc.flush():
        chunks.append(payload.payload)
    enc.close()

    decoded = decode_annexb(b"".join(chunks))
    if not decoded:
        return False
    return all(f.width == width and f.height == height for f in decoded)

nvenc_gpu_pdum

PyAV-free GPU H.264 encoder via the NVENC SDK (pdum.nvenc).

The third GPU path, and the only PyAV-free one. Where :class:~pdum.rfb.encoders.nvenc.NvencCpuEncoder uploads a host rgb24 frame and :class:~pdum.rfb.encoders.nvenc_gpu_pyav.NvencGpuPyavEncoder needs PyAV ≥ 18, this backend hands a GPU-resident NV12 buffer straight to NVIDIA's NvEncoderCuda (the habemus-papadum-nvenc package, import pdum.nvenc) and gets Annex B back — no ffmpeg/PyAV anywhere in the encode path. It is the fastest path measured on the test hardware (see docs/performance.md).

Input handling mirrors the zero-copy CUDA backend:

  • a CUDA nv12 frame — encoded as-is (the true zero-copy case);
  • a CUDA rgb24/rgba8 frame — converted to NV12 on the GPU first (:func:pdum.rfb.gpu.rgb_to_nv12);
  • a host rgb24/rgba8 frame — uploaded then converted (graceful fallback).

Gate on :func:nvenc_gpu_pdum_available before constructing one. Emits the same low-latency Annex B as the other H.264 backends, so the wire format, forced-keyframe handling, and payload packing are unchanged. Fixed-resolution: a resize rebuilds the encoder (the session does this via its encoder_factory) and forces a keyframe.

NvencGpuPdumEncoder

Encode CUDA (or host, with upload) frames to H.264 Annex B via the NVENC SDK.

Same constructor / :class:~pdum.rfb.types.EncoderBackend interface as the other H.264 backends. Frames narrower than :data:~pdum.rfb.encoders.nvenc.NVENC_MIN_WIDTH are rejected (NVENC cannot open below it) and dimensions must be even (NV12).

Source code in src/pdum/rfb/encoders/nvenc_gpu_pdum.py 
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
class NvencGpuPdumEncoder:
    """Encode CUDA (or host, with upload) frames to H.264 Annex B via the NVENC SDK.

    Same constructor / :class:`~pdum.rfb.types.EncoderBackend` interface as the other
    H.264 backends. Frames narrower than
    :data:`~pdum.rfb.encoders.nvenc.NVENC_MIN_WIDTH` are rejected (NVENC cannot open
    below it) and dimensions must be even (NV12).
    """

    encoder_label = "nvenc-gpu-pdum"

    def __init__(
        self,
        *,
        width: int,
        height: int,
        fps: int = 30,
        bitrate: int = 12_000_000,
        codec_string: str | None = None,
        preset: str = "p4",
        tuning: str = "ll",
        pipeline_depth: int = 0,
    ) -> None:
        if width < NVENC_MIN_WIDTH:
            raise ValueError(
                f"NVENC requires width >= {NVENC_MIN_WIDTH}; got {width}. "
                "Use a larger framebuffer or fall back to the libx264 encoder."
            )
        if width % 2 or height % 2:
            raise ValueError(f"NV12 requires even dimensions; got {width}x{height}")

        import cupy as cp
        from pdum.nvenc import NvencEncoder

        self.width = width
        self.height = height
        self.fps = fps
        self.bitrate = bitrate
        self.codec_string = codec_string or DEFAULT_H264_CODEC
        # NVENC chooses the H.264 profile (High by default), which may not match `codec_string`.
        # We correct it to the real profile/level from the first SPS (see _refresh_codec_string)
        # so the browser's per-chunk VideoDecoder.configure() matches the bitstream; _codec_locked
        # guards that one-time update.
        self._codec_locked = False
        self.frame_index = 0
        self._duration_us = int(1_000_000 / fps)
        # pipeline_depth > 0 selects the token-based pipelined path (submit()/flush_pipeline);
        # 0 is the synchronous 1-in-1-out default. On NVENC this is the backend where it pays
        # off: it maps to the SDK's extra_output_delay, so several frames stay in flight and
        # encode overlaps render/convert for throughput (at depth/fps of extra latency). See
        # docs/pipelined_encode.md.
        self.pipeline_depth = max(0, int(pipeline_depth))
        self._pending_ts: dict[int, int] = {}  # seq -> timestamp_us for in-flight frames
        # Reusable NV12 staging buffer for the rgb/host input paths. ll tuning with no
        # lookahead consumes each frame before the next, so a single buffer is safe; under
        # pipelining CopyToDeviceFrame copies it into NVENC's own input slot before submit()
        # returns (the deviceSynchronize() in encode() guarantees the NV12 is ready first).
        self._nv12 = cp.empty((height + height // 2, width), cp.uint8)
        # cuda_context=0 -> retain the device *primary* context (the one CuPy uses), so
        # CuPy device pointers are valid to NVENC with no cross-context copy.
        # profile="baseline": NVENC defaults to High, but hardware WebCodecs H.264 decoders
        # (the browser's real decode path) are stricter than software ones and can silently
        # fail on High — so we emit Constrained/Baseline (profile_idc 66) to match the libx264
        # and PyAV-NVENC backends, which the browser decodes reliably. The codec string is then
        # derived from the actual SPS (see _refresh_codec_string), so it stays truthful.
        self._enc = NvencEncoder(
            width,
            height,
            codec="h264",
            preset=preset,
            tuning=tuning,
            fps=fps,
            gop=fps,
            bitrate=bitrate,
            extra_output_delay=self.pipeline_depth,
            profile="baseline",
        )

    def _packed_nv12(self, frame: RawFrame):
        """Return a contiguous CUDA NV12 ``(H+H//2, W)`` buffer for ``frame``."""
        import cupy as cp

        from ..gpu import rgb_to_nv12

        if frame.memory == "cuda" and frame.pixel_format == "nv12":
            packed = cp.ascontiguousarray(cp.asarray(frame.data))
            if packed.shape != self._nv12.shape:
                raise ValueError(f"nv12 frame shape {packed.shape!r} != encoder {self._nv12.shape!r}")
            return packed
        if frame.pixel_format not in ("rgb24", "rgba8"):
            raise TypeError(f"NvencGpuPdumEncoder cannot encode {frame.pixel_format!r} frames")
        # rgb24/rgba8, CUDA or host: convert (uploading first if host) into the buffer.
        return rgb_to_nv12(frame.data, out=self._nv12)

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        import cupy as cp

        packed = self._packed_nv12(frame)
        # Ensure the NV12 (kernel/upload above) is complete before NVENC's intra-GPU
        # copy reads it. Sub-ms at these sizes; keeps the path correct across streams.
        cp.cuda.runtime.deviceSynchronize()
        self.frame_index += 1
        if self.pipeline_depth > 0:
            return self._encode_pipelined(packed, frame.seq, frame.timestamp_us, force_keyframe)
        data = self._enc.encode(packed, force_idr=force_keyframe)
        if not data:
            return []
        self._refresh_codec_string(data)
        keyframe = force_keyframe or _contains_idr(data)
        return [self._payload(frame.seq, frame.timestamp_us, data, keyframe)]

    def _encode_pipelined(self, packed, seq: int, timestamp_us: int, force_keyframe: bool) -> list[EncodedPayload]:
        """Submit one frame without waiting; return whatever AUs are ready, each labeled with
        its *recovered* seq (the frame it actually encoded), not this call's seq. The recovered
        keyframe comes straight from NVENC's pictureType. See docs/pipelined_encode.md."""
        self._pending_ts[seq] = timestamp_us
        aus = self._enc.submit(packed, seq, force_idr=force_keyframe)
        payloads: list[EncodedPayload] = []
        for s, data, key in aus:
            self._refresh_codec_string(data)  # first emitted AU (frame 0) carries the SPS
            payloads.append(self._payload(s, self._pending_ts.pop(s, timestamp_us), data, key))
        return payloads

    def _refresh_codec_string(self, annexb: bytes) -> None:
        """Once the first SPS is seen, replace the placeholder codec string with the profile/level
        NVENC actually emitted, so every payload (and the browser's decoder) matches the bitstream.
        Idempotent after the first keyframe; a no-op on delta chunks (no SPS)."""
        if self._codec_locked:
            return
        derived = _codec_string_from_annexb(annexb)
        if derived is not None:
            self.codec_string = derived
            self._codec_locked = True

    def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
        """Settled-scene still: a forced **IDR** of the resting frame (see the CPU
        backend's :meth:`~pdum.rfb.encoders.h264_cpu.H264CpuEncoder.encode_still`)."""
        return self.encode(frame, force_keyframe=True)

    def flush(self) -> list[EncodedPayload]:
        if self.pipeline_depth > 0:
            aus = self._enc.flush_pipeline()
            return [self._payload(s, self._pending_ts.pop(s, 0), data, key) for s, data, key in aus]
        data = self._enc.flush()
        if not data:
            return []
        return [self._payload(-1, 0, data, _contains_idr(data))]

    def close(self) -> None:
        try:
            self._enc.close()
        except Exception:  # pragma: no cover - encoder may already be closed
            pass

    def _payload(self, seq: int, timestamp_us: int, data: bytes, keyframe: bool) -> EncodedPayload:
        return EncodedPayload(
            seq=seq,
            kind="video",
            timestamp_us=timestamp_us,
            width=self.width,
            height=self.height,
            payload=bytes(data),
            codec=self.codec_string,
            keyframe=keyframe,
            duration_us=self._duration_us,
            metadata={"bitstream": "annexb", "encoder": self.encoder_label},
        )
encode_still(frame)

Settled-scene still: a forced IDR of the resting frame (see the CPU backend's :meth:~pdum.rfb.encoders.h264_cpu.H264CpuEncoder.encode_still).

Source code in src/pdum/rfb/encoders/nvenc_gpu_pdum.py 
198
199
200
201
def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
    """Settled-scene still: a forced **IDR** of the resting frame (see the CPU
    backend's :meth:`~pdum.rfb.encoders.h264_cpu.H264CpuEncoder.encode_still`)."""
    return self.encode(frame, force_keyframe=True)

nvenc_gpu_pdum_available() cached

True if the PyAV-free SDK NVENC path is usable in this process (cached).

Checks CuPy + pdum.nvenc importable, then actually opens an NVENC session and encodes two frames (no decode, so it stays PyAV-free). Runs at most once per process because it opens a real encoder session.

Source code in src/pdum/rfb/encoders/nvenc_gpu_pdum.py 
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
@functools.lru_cache(maxsize=1)
def nvenc_gpu_pdum_available() -> bool:
    """True if the PyAV-free SDK NVENC path is usable in this process (cached).

    Checks CuPy + ``pdum.nvenc`` importable, then actually opens an NVENC session and
    encodes two frames (no decode, so it stays PyAV-free). Runs at most once per
    process because it opens a real encoder session.
    """
    if importlib.util.find_spec("cupy") is None or importlib.util.find_spec("pdum.nvenc") is None:
        return False
    try:
        import cupy as cp

        from ..gpu import cuda_frame

        enc = NvencGpuPdumEncoder(width=256, height=128, fps=4, bitrate=2_000_000)
        total = 0
        for seq in range(2):
            nv12 = cp.zeros((128 + 64, 256), cp.uint8)
            frame = cuda_frame(nv12, pixel_format="nv12", height=128, seq=seq)
            total += sum(len(p.payload) for p in enc.encode(frame, force_keyframe=(seq == 0)))
        total += sum(len(p.payload) for p in enc.flush())
        enc.close()
        return total > 0
    except Exception as exc:
        # cupy + pdum.nvenc import fine (guarded above), yet the encode probe failed: the
        # package is *installed but unusable*. The usual cause is a stale native build — the
        # compiled .so drifted from packages/nvenc/src/cpp after a C++ edit without a version
        # bump, so a later `uv sync` reinstalled the cached wheel over a fresh one (e.g. the
        # wrapper passes a constructor arg the old .so doesn't accept -> TypeError). Log it so
        # a silently greyed-out backend in `pdum-rfb demo` / doctor / serve() auto-select
        # becomes an actionable hint instead of the misleading "package not installed".
        logging.getLogger(__name__).warning(
            "pdum.nvenc is installed but its NVENC probe failed (%s: %s) — likely a stale "
            "native build; rebuild with scripts/rebuild-nvenc.sh",
            type(exc).__name__,
            exc,
        )
        return False

self_test(width=256, height=256, frames=8)

Encode synthetic CUDA NV12 frames via the SDK and decode them back (needs PyAV).

Source code in src/pdum/rfb/encoders/nvenc_gpu_pdum.py 
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
def self_test(width: int = 256, height: int = 256, frames: int = 8) -> bool:
    """Encode synthetic CUDA NV12 frames via the SDK and decode them back (needs PyAV)."""
    if not nvenc_gpu_pdum_available():
        return False

    import cupy as cp

    from ..gpu import cuda_frame
    from ..testing import decode_annexb

    width = max(width, NVENC_MIN_WIDTH)
    enc = NvencGpuPdumEncoder(width=width, height=height, fps=int(frames))
    chunks: list[bytes] = []
    for seq in range(frames):
        nv12 = cp.empty((height + height // 2, width), cp.uint8)
        nv12[:height] = (seq * 7) % 256  # moving luma
        nv12[height:] = 128  # neutral chroma
        frame = cuda_frame(nv12, pixel_format="nv12", height=height, seq=seq)
        for payload in enc.encode(frame, force_keyframe=(seq == 0)):
            chunks.append(payload.payload)
    for payload in enc.flush():
        chunks.append(payload.payload)
    enc.close()

    decoded = decode_annexb(b"".join(chunks))
    if not decoded:
        return False
    return all(f.width == width and f.height == height for f in decoded)

nvenc_gpu_pyav

Zero-copy CUDA → NVENC H.264 encoder (the roadmap's GPU-buffer path).

Unlike :class:~pdum.rfb.encoders.nvenc.NvencCpuEncoder — which takes a host rgb24 frame, reformats it to yuv420p on the CPU, and lets NVENC upload it — this backend encodes a CUDA-resident frame directly: a CuPy / DLPack NV12 buffer is handed to h264_nvenc via av.VideoFrame.from_dlpack with no host round-trip. On the tested Ada laptop GPU that is ~2.4–4.3× lower per-frame latency (1080p 2.5 ms vs 7.3 ms; 4K 7.1 ms vs 30.5 ms) and frees the CPU entirely — see docs/gpu_zerocopy.md.

It accepts:

  • a CUDA nv12 frame (RawFrame(memory="cuda", pixel_format="nv12")) — the true zero-copy case; the device buffer is encoded as-is;
  • a CUDA rgb24/rgba8 frame — converted to NV12 on the GPU first (:func:pdum.rfb.gpu.rgb_to_nv12, ~0.01 ms at 1080p);
  • a host rgb24/rgba8 frame — uploaded then converted (graceful fallback so a gpu=True server still works if the publisher pushes a host frame).

Requires PyAV ≥ 18 (or a from-source build with the CUDA-encode fix). Gate on :func:pdum.rfb.gpu.cuda_zerocopy_available before constructing one. Emits the same low-latency Annex B as the other H.264 backends, so the wire format, forced-keyframe handling, and payload packing are inherited unchanged.

NvencGpuPyavEncoder

Bases: H264CpuEncoder

Encode CUDA (or host, with upload) frames to H.264 Annex B via NVENC, zero-copy.

Drop-in for :class:~pdum.rfb.encoders.nvenc.NvencCpuEncoder (same constructor / EncoderBackend interface); only the input handling differs. Frames narrower than :data:~pdum.rfb.encoders.nvenc.NVENC_MIN_WIDTH are rejected (NVENC cannot open below it), and dimensions must be even (NV12).

Source code in src/pdum/rfb/encoders/nvenc_gpu_pyav.py 
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
class NvencGpuPyavEncoder(H264CpuEncoder):
    """Encode CUDA (or host, with upload) frames to H.264 Annex B via NVENC, zero-copy.

    Drop-in for :class:`~pdum.rfb.encoders.nvenc.NvencCpuEncoder` (same
    constructor / ``EncoderBackend`` interface); only the input handling differs.
    Frames narrower than :data:`~pdum.rfb.encoders.nvenc.NVENC_MIN_WIDTH` are
    rejected (NVENC cannot open below it), and dimensions must be even (NV12).
    """

    encoder_label = "nvenc-gpu-pyav"

    def __init__(
        self,
        *,
        width: int,
        height: int,
        fps: int = 30,
        bitrate: int = 12_000_000,
        codec_string: str | None = None,
    ) -> None:
        if width < NVENC_MIN_WIDTH:
            raise ValueError(
                f"NVENC requires width >= {NVENC_MIN_WIDTH}; got {width}. "
                "Use a larger framebuffer or fall back to the libx264 encoder."
            )
        if width % 2 or height % 2:
            raise ValueError(f"NV12 requires even dimensions; got {width}x{height}")
        # Share CuPy's primary CUDA context with FFmpeg (must precede CuPy use; a
        # no-op if the caller already did it at startup, as recommended).
        enable_cuda_context_sharing()
        self._cctx = None
        self._nv12 = None  # reusable NV12 staging buffer for the rgb/host paths
        super().__init__(
            width=width,
            height=height,
            fps=fps,
            bitrate=bitrate,
            codec_string=codec_string or DEFAULT_H264_CODEC,
        )

    def _make_context(self):
        import av
        import cupy as cp
        from av.video.frame import CudaContext

        # One persistent CUDA context shared by every from_dlpack frame and the
        # encoder, so all frames carry the same hw_frames_ctx.
        self._cctx = CudaContext(device_id=0, primary_ctx=True)
        self._nv12 = cp.empty((self.height + self.height // 2, self.width), cp.uint8)

        ctx = av.CodecContext.create(_NVENC_CODEC, "w")
        ctx.width = self.width
        ctx.height = self.height
        ctx.pix_fmt = "cuda"  # the key difference: GPU-resident input
        ctx.time_base = Fraction(1, self.fps)
        ctx.framerate = Fraction(self.fps, 1)
        ctx.bit_rate = self.bitrate
        # Same low-latency config as the host NVENC backend (see its docstring for
        # the rc=vbr / profile=baseline rationale).
        ctx.options = {
            "preset": "p4",
            "tune": "ll",
            "rc": "vbr",
            "profile": "baseline",
            "bf": "0",
            "delay": "0",
            "forced-idr": "1",
            "g": str(self.fps),
        }
        return ctx

    def _packed_nv12(self, frame):
        """Return a contiguous CUDA NV12 ``(H+H//2, W)`` buffer for ``frame``."""
        import cupy as cp

        if frame.memory == "cuda" and frame.pixel_format == "nv12":
            packed = cp.ascontiguousarray(cp.asarray(frame.data))
            if packed.shape != self._nv12.shape:
                raise ValueError(f"nv12 frame shape {packed.shape!r} != encoder {self._nv12.shape!r}")
            return packed
        if frame.pixel_format not in ("rgb24", "rgba8"):
            raise TypeError(f"NvencGpuPyavEncoder cannot encode {frame.pixel_format!r} frames")
        # rgb24/rgba8, CUDA or host: convert (uploading first if host) into the
        # reusable staging buffer. delay=0 means the prior frame is fully consumed
        # before we overwrite, so a single reused buffer is safe.
        return rgb_to_nv12(frame.data, out=self._nv12)

    def encode(self, frame, *, force_keyframe: bool = False):
        import av

        packed = self._packed_nv12(frame)
        y, uv = nv12_planes(packed)
        vf = av.VideoFrame.from_dlpack(
            [y, uv],
            format="nv12",
            width=self.width,
            height=self.height,
            primary_ctx=True,
            cuda_context=self._cctx,
        )
        vf.pts = self.frame_index
        vf.time_base = Fraction(1, self.fps)
        if force_keyframe:
            vf.pict_type = av.video.frame.PictureType.I
        self.frame_index += 1
        return [self._payload(frame.seq, frame.timestamp_us, pkt) for pkt in self._drain(vf)]

nvenc_gpu_pyav_available()

True if the zero-copy CUDA→NVENC path is usable (see :func:cuda_zerocopy_available).

Source code in src/pdum/rfb/encoders/nvenc_gpu_pyav.py 
36
37
38
def nvenc_gpu_pyav_available() -> bool:
    """True if the zero-copy CUDA→NVENC path is usable (see :func:`cuda_zerocopy_available`)."""
    return cuda_zerocopy_available()

self_test(width=256, height=256, frames=8)

Encode a few synthetic CUDA NV12 frames via zero-copy NVENC and decode back.

Source code in src/pdum/rfb/encoders/nvenc_gpu_pyav.py 
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
def self_test(width: int = 256, height: int = 256, frames: int = 8) -> bool:
    """Encode a few synthetic CUDA NV12 frames via zero-copy NVENC and decode back."""
    if not nvenc_gpu_pyav_available():
        return False

    import cupy as cp

    from ..gpu import cuda_frame
    from ..testing import decode_annexb

    width = max(width, NVENC_MIN_WIDTH)
    enc = NvencGpuPyavEncoder(width=width, height=height, fps=int(frames))
    chunks: list[bytes] = []
    for seq in range(frames):
        nv12 = cp.empty((height + height // 2, width), cp.uint8)
        nv12[:height] = (seq * 7) % 256  # moving luma
        nv12[height:] = 128  # neutral chroma
        frame = cuda_frame(nv12, pixel_format="nv12", height=height, seq=seq)
        for payload in enc.encode(frame, force_keyframe=(seq == 0)):
            chunks.append(payload.payload)
    for payload in enc.flush():
        chunks.append(payload.payload)
    enc.close()

    decoded = decode_annexb(b"".join(chunks))
    if not decoded:
        return False
    return all(f.width == width and f.height == height for f in decoded)

vtenc

macOS H.264 EncoderBackend via Apple VideoToolbox (pdum.vtenc).

The rfb wrapper around the habemus-papadum-vtenc package (import pdum.vtenc): it adapts VtEncoder (host NV12 → H.264 Annex B) to the :class:~pdum.rfb.types.EncoderBackend protocol so it slots into the same registry/transport seam as the libx264 and NVENC backends. Host rgb24/rgba8 frames are converted to NV12 on the CPU (BT.601 limited range, matching :func:pdum.rfb.gpu.rgb_to_nv12); an already-NV12 frame passes straight through. Emits the same low-latency Annex B (in-band SPS/PPS, no reordering), so the wire format and the browser are unchanged.

The advertised codec string is taken from the actual emitted SPS (VtEncoder.codec_string, e.g. avc1.420028 at 1080p), not the constant avc1.42E01F — VideoToolbox picks the level from the resolution.

Gate on :func:vtenc_available before constructing one. Registered as "vtenc".

VideoToolboxEncoder

Encode host frames to H.264 Annex B via Apple VideoToolbox.

Same constructor / :class:~pdum.rfb.types.EncoderBackend interface as the other H.264 backends. Dimensions must be even (NV12).

Source code in src/pdum/rfb/encoders/vtenc.py 
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
class VideoToolboxEncoder:
    """Encode host frames to H.264 Annex B via Apple VideoToolbox.

    Same constructor / :class:`~pdum.rfb.types.EncoderBackend` interface as the other H.264
    backends. Dimensions must be even (NV12).
    """

    encoder_label = "vtenc"

    def __init__(
        self,
        *,
        width: int,
        height: int,
        fps: int = 30,
        bitrate: int = 12_000_000,
        codec_string: str | None = None,
        pipeline_depth: int = 0,
    ) -> None:
        if width % 2 or height % 2:
            raise ValueError(f"NV12 requires even dimensions; got {width}x{height}")

        from pdum.vtenc import VtEncoder

        self.width = width
        self.height = height
        self.fps = fps
        self.bitrate = bitrate
        self.codec_string = codec_string or DEFAULT_H264_CODEC
        # pipeline_depth > 0 selects the token-based pipelined path (submit()/flush_pipeline);
        # 0 is the synchronous 1-in-1-out default. NOTE: on VideoToolbox specifically this is
        # *correct but not faster* — low-latency RC is synchronous (measured no throughput win,
        # see docs/pipelined_encode.md). The knob exists for the NVENC backend where it pays
        # off; VT exercises the same recovered-seq path for correctness/parity.
        self.pipeline_depth = max(0, int(pipeline_depth))
        self._pending_ts: dict[int, int] = {}  # seq -> timestamp_us for in-flight frames
        self._duration_us = int(1_000_000 / fps)
        self._nv12 = np.empty((height + height // 2, width), np.uint8)
        self._enc = VtEncoder(width, height, codec="h264", fps=fps, gop=fps, bitrate=bitrate)

    def _packed_nv12(self, frame: RawFrame) -> np.ndarray:
        data = frame.data
        if frame.memory == "metal":
            return self._packed_nv12_metal(frame)
        if frame.pixel_format == "nv12":
            arr = np.ascontiguousarray(np.asarray(data))
            if arr.shape != self._nv12.shape:
                raise ValueError(f"nv12 frame shape {arr.shape!r} != encoder {self._nv12.shape!r}")
            return arr
        if frame.pixel_format not in ("rgb24", "rgba8"):
            raise TypeError(f"VideoToolboxEncoder cannot encode {frame.pixel_format!r} frames")
        return _host_rgb_to_nv12(np.asarray(data), self._nv12)

    def _packed_nv12_metal(self, frame: RawFrame) -> np.ndarray:
        """Metal (MLX) frame: convert RGB(A)→NV12 on the GPU, then hand the binding a host NV12
        view (unified memory → near-zero-copy). Avoids the ~6.6 ms/1080p CPU color conversion."""
        from ..metal import rgb_to_nv12 as _mlx_rgb_to_nv12
        from ..metal import to_host_nv12

        if frame.pixel_format == "nv12":
            arr = to_host_nv12(frame.data)
            if arr.shape != self._nv12.shape:
                raise ValueError(f"nv12 frame shape {arr.shape!r} != encoder {self._nv12.shape!r}")
            return arr
        if frame.pixel_format not in ("rgb24", "rgba8"):
            raise TypeError(f"VideoToolboxEncoder cannot encode {frame.pixel_format!r} Metal frames")
        return to_host_nv12(_mlx_rgb_to_nv12(frame.data))

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        packed = self._packed_nv12(frame)
        if self.pipeline_depth > 0:
            return self._encode_pipelined(packed, frame.seq, frame.timestamp_us, force_keyframe)
        data = self._enc.encode(packed, force_idr=force_keyframe)
        if not data:
            return []
        # VideoToolbox derives the level from the resolution; report the real SPS string.
        self.codec_string = self._enc.codec_string or self.codec_string
        keyframe = force_keyframe or _contains_idr(data)
        return [self._payload(frame.seq, frame.timestamp_us, data, keyframe)]

    def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
        """Encode ``frame`` as a self-contained keyframe (a clean IDR) for "still after settle".

        VideoToolbox has no lossless mode, so the video-path still is a forced IDR — the same
        contract as the libx264/NVENC backends. Without this method the session's still gate
        (``hasattr(encoder, "encode_still")``) is False and stills silently no-op on macOS.
        """
        return self.encode(frame, force_keyframe=True)

    def _encode_pipelined(self, packed, seq: int, timestamp_us: int, force_keyframe: bool) -> list[EncodedPayload]:
        """Submit one frame without waiting; return whatever AUs are ready, each labeled with
        its *recovered* seq (the frame it actually encoded), not this call's seq. See
        docs/pipelined_encode.md and docs/proposals/completed/encoder_sync_and_seq_attribution.md."""
        self._pending_ts[seq] = timestamp_us
        aus = self._enc.submit(packed, seq, force_idr=force_keyframe)
        self.codec_string = self._enc.codec_string or self.codec_string
        return [self._payload(s, self._pending_ts.pop(s, timestamp_us), data, key) for s, data, key in aus]

    def flush(self) -> list[EncodedPayload]:
        if self.pipeline_depth > 0:
            aus = self._enc.flush_pipeline()
            return [self._payload(s, self._pending_ts.pop(s, 0), data, key) for s, data, key in aus]
        data = self._enc.flush()
        if not data:
            return []
        return [self._payload(-1, 0, data, _contains_idr(data))]

    def close(self) -> None:
        try:
            self._enc.close()
        except Exception:  # pragma: no cover - encoder may already be closed
            pass

    def _payload(self, seq: int, timestamp_us: int, data: bytes, keyframe: bool) -> EncodedPayload:
        return EncodedPayload(
            seq=seq,
            kind="video",
            timestamp_us=timestamp_us,
            width=self.width,
            height=self.height,
            payload=bytes(data),
            codec=self.codec_string,
            keyframe=keyframe,
            duration_us=self._duration_us,
            metadata={"bitstream": "annexb", "encoder": self.encoder_label},
        )
encode_still(frame)

Encode frame as a self-contained keyframe (a clean IDR) for "still after settle".

VideoToolbox has no lossless mode, so the video-path still is a forced IDR — the same contract as the libx264/NVENC backends. Without this method the session's still gate (hasattr(encoder, "encode_still")) is False and stills silently no-op on macOS.

Source code in src/pdum/rfb/encoders/vtenc.py 
139
140
141
142
143
144
145
146
def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
    """Encode ``frame`` as a self-contained keyframe (a clean IDR) for "still after settle".

    VideoToolbox has no lossless mode, so the video-path still is a forced IDR — the same
    contract as the libx264/NVENC backends. Without this method the session's still gate
    (``hasattr(encoder, "encode_still")``) is False and stills silently no-op on macOS.
    """
    return self.encode(frame, force_keyframe=True)

self_test(width=256, height=192, frames=8)

Encode synthetic host frames through VideoToolbox and decode them back (needs PyAV).

Source code in src/pdum/rfb/encoders/vtenc.py 
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
def self_test(width: int = 256, height: int = 192, frames: int = 8) -> bool:
    """Encode synthetic host frames through VideoToolbox and decode them back (needs PyAV)."""
    if not vtenc_available():
        return False
    from ..testing import decode_annexb, render_test_pattern

    enc = VideoToolboxEncoder(width=width, height=height, fps=int(frames))
    chunks: list[bytes] = []
    for seq in range(frames):
        frame = RawFrame(seq, width, height, seq * 1000, "rgb24", "cpu", render_test_pattern(seq, width, height))
        for payload in enc.encode(frame, force_keyframe=(seq == 0)):
            chunks.append(payload.payload)
    for payload in enc.flush():
        chunks.append(payload.payload)
    enc.close()
    decoded = decode_annexb(b"".join(chunks))
    return bool(decoded) and all(f.width == width and f.height == height for f in decoded)

vtenc_available() cached

True if the VideoToolbox backend is usable in this process (cached).

macOS + pdum.vtenc importable + VideoToolbox can open an H.264 session.

Source code in src/pdum/rfb/encoders/vtenc.py 
187
188
189
190
191
192
193
194
195
196
197
198
199
200
@functools.lru_cache(maxsize=1)
def vtenc_available() -> bool:
    """True if the VideoToolbox backend is usable in this process (cached).

    macOS + ``pdum.vtenc`` importable + VideoToolbox can open an H.264 session.
    """
    if sys.platform != "darwin" or importlib.util.find_spec("pdum.vtenc") is None:
        return False
    try:
        from pdum.vtenc import supported

        return bool(supported())
    except Exception:
        return False

gpu

GPU zero-copy helpers for the NVENC CUDA path (DLPack in, no host copy).

This module lets a CUDA-resident frame (a CuPy / PyTorch / any __dlpack__ or __cuda_array_interface__ tensor) be encoded by NVENC without a round-trip through host memory — the encoder reads the device buffer directly. It is the zero-copy counterpart to :mod:pdum.rfb.encoders.nvenc_cpu (which uploads host rgb24 and reformats to yuv420p on the CPU first).

Everything here lazy-imports CuPy, so import pdum.rfb.gpu is always safe; the functions raise only when actually called without CuPy.

Requirements (see :func:cuda_zerocopy_available)
  1. CuPy (cupy-cuda13x / cupy-cuda12x; cp314 wheels exist).
  2. An NVENC-capable GPU + driver (same gate as the host NVENC backend).
  3. PyAV that can encode CUDA frames. from_dlpack (frame creation) landed in PyAV 17.0, but feeding those frames to an encoder (hw_frames_ctx adopted before avcodec_open2) lands only in PyAV 18.0 (unreleased as of this writing; the fix is on main — issue #2199). On PyAV 17.x this raises "hw_frames_ctx must be set when using GPU frames as input"; there is no pure-Python workaround (PyAV exposes no handle to set avctx->hw_frames_ctx), so a < 18 install must build PyAV from source (main or the small patch documented in docs/gpu_zerocopy.md).
Two non-obvious gotchas this module handles for you
  • One shared CUDA context. CuPy uses the device primary context; FFmpeg's CUDA hwcontext (primary_ctx=True) wants it created with CU_CTX_SCHED_BLOCKING_SYNC flags. If CuPy activates it first with the default (auto) flags, primary_ctx=True fails ("incompatible flags") and a separate (primary_ctx=False) context can't register CuPy's pointers ("resource register failed"). :func:enable_cuda_context_sharing pre-sets the flags — call it before any CuPy/Torch CUDA work (importing is fine; the first allocation/op is what activates the context).
  • NV12 must be one contiguous allocation (Y plane then UV plane), because NVENC reads UV at base + pitch*height. :func:rgb_to_nv12 produces that layout; :func:nv12_planes slices it back into the two DLPack planes.

HostFrameAdapter

Wrap a host :class:~pdum.rfb.types.EncoderBackend so it tolerates CUDA frames.

The image / CPU encoders expect host frames; when the publisher pushes CUDA frames (serve(gpu=True)), an image-transport viewer's encoder is wrapped in this adapter, which downloads each CUDA frame to host rgb24 (:func:to_host_rgb) before delegating. Host frames pass through untouched, so the wrapped encoder stays dependency-pure and the existing contract is intact.

Source code in src/pdum/rfb/gpu.py 
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
class HostFrameAdapter:
    """Wrap a host :class:`~pdum.rfb.types.EncoderBackend` so it tolerates CUDA frames.

    The image / CPU encoders expect host frames; when the publisher pushes CUDA
    frames (``serve(gpu=True)``), an image-transport viewer's encoder is wrapped in
    this adapter, which downloads each CUDA frame to host ``rgb24``
    (:func:`to_host_rgb`) before delegating. Host frames pass through untouched, so
    the wrapped encoder stays dependency-pure and the existing contract is intact.
    """

    __slots__ = ("_inner",)

    def __init__(self, inner: Any) -> None:
        self._inner = inner

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False):
        return self._inner.encode(self._to_host(frame), force_keyframe=force_keyframe)

    def encode_still(self, frame: RawFrame):
        """Download a CUDA frame to host, then delegate the still to the wrapped
        encoder (keeps "still after settle" working in ``serve(gpu=True)`` +
        image-transport viewers)."""
        return self._inner.encode_still(self._to_host(frame))

    @staticmethod
    def _to_host(frame: RawFrame) -> RawFrame:
        if frame.memory == "cpu":
            return frame
        import dataclasses

        return dataclasses.replace(frame, data=to_host_rgb(frame), memory="cpu", pixel_format="rgb24")

    def flush(self):
        return self._inner.flush()

    def close(self) -> None:
        self._inner.close()

encode_still(frame)

Download a CUDA frame to host, then delegate the still to the wrapped encoder (keeps "still after settle" working in serve(gpu=True) + image-transport viewers).

Source code in src/pdum/rfb/gpu.py 
301
302
303
304
305
def encode_still(self, frame: RawFrame):
    """Download a CUDA frame to host, then delegate the still to the wrapped
    encoder (keeps "still after settle" working in ``serve(gpu=True)`` +
    image-transport viewers)."""
    return self._inner.encode_still(self._to_host(frame))

cuda_frame(array, *, pixel_format='auto', width=None, height=None, seq=0, timestamp_us=0)

Wrap a device tensor as a CUDA :class:~pdum.rfb.types.RawFrame for publish().

pixel_format="auto" infers from shape: (H, W, 3) -> rgb24, (H, W, 4) -> rgba8, 2-D (H+H//2, W) -> nv12. Pass an explicit pixel_format (and height for ambiguous NV12) to override. The tensor is referenced, not copied; keep it alive until the frame is encoded.

Source code in src/pdum/rfb/gpu.py 
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
def cuda_frame(
    array: Any,
    *,
    pixel_format: str = "auto",
    width: int | None = None,
    height: int | None = None,
    seq: int = 0,
    timestamp_us: int = 0,
) -> RawFrame:
    """Wrap a device tensor as a CUDA :class:`~pdum.rfb.types.RawFrame` for ``publish()``.

    ``pixel_format="auto"`` infers from shape: ``(H, W, 3)`` -> ``rgb24``,
    ``(H, W, 4)`` -> ``rgba8``, 2-D ``(H+H//2, W)`` -> ``nv12``. Pass an explicit
    ``pixel_format`` (and ``height`` for ambiguous NV12) to override. The tensor
    is referenced, not copied; keep it alive until the frame is encoded.
    """
    arr = _as_cupy(array)
    if pixel_format == "auto":
        if arr.ndim == 3 and arr.shape[2] == 3:
            pixel_format = "rgb24"
        elif arr.ndim == 3 and arr.shape[2] == 4:
            pixel_format = "rgba8"
        elif arr.ndim == 2:
            pixel_format = "nv12"
        else:
            raise ValueError(f"cannot infer pixel_format from shape {arr.shape!r}")
    if pixel_format == "nv12":
        width = int(arr.shape[1]) if width is None else width
        height = nv12_height(arr) if height is None else height
    else:
        height = int(arr.shape[0]) if height is None else height
        width = int(arr.shape[1]) if width is None else width
    return RawFrame(
        seq=seq,
        width=int(width),
        height=int(height),
        timestamp_us=int(timestamp_us),
        pixel_format=pixel_format,  # type: ignore[arg-type]
        memory="cuda",
        data=arr,
    )

cuda_zerocopy_available() cached

True if the zero-copy CUDA→NVENC path is usable in this process (cached).

Checks, in order: CuPy importable; an NVENC-capable GPU/driver (:func:pdum.rfb.encoders.nvenc_cpu.nvenc_cpu_available); and that PyAV can actually encode a CUDA frame (PyAV ≥ 18 or a from-source build with the fix). The self-test opens an NVENC session, so it runs at most once per process.

.. note:: Call :func:enable_cuda_context_sharing before any CuPy CUDA op for a reliable result — if CuPy has already activated the primary context with the default flags, the shared-context probe (and the encoder) will fail.

Source code in src/pdum/rfb/gpu.py 
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
@functools.lru_cache(maxsize=1)
def cuda_zerocopy_available() -> bool:
    """True if the zero-copy CUDA→NVENC path is usable in this process (cached).

    Checks, in order: CuPy importable; an NVENC-capable GPU/driver
    (:func:`pdum.rfb.encoders.nvenc_cpu.nvenc_cpu_available`); and that PyAV can actually
    encode a CUDA frame (PyAV ≥ 18 or a from-source build with the fix). The
    self-test opens an NVENC session, so it runs at most once per process.

    .. note::
       Call :func:`enable_cuda_context_sharing` **before any CuPy CUDA op** for a
       reliable result — if CuPy has already activated the primary context with
       the default flags, the shared-context probe (and the encoder) will fail.
    """
    if importlib.util.find_spec("cupy") is None:
        return False
    try:
        from .encoders.nvenc_cpu import nvenc_cpu_available

        if not nvenc_cpu_available():
            return False
    except Exception:
        return False
    return _selftest_zerocopy_encode()

enable_cuda_context_sharing(device_id=0, *, sched=_FFMPEG_SCHED)

Pre-set the device primary-context flags so CuPy and FFmpeg share one context.

Call this once, first thing in your program, before any CuPy/PyTorch CUDA op. It sets the device's primary-context scheduling flags to what FFmpeg's CUDA hwcontext wants (CU_CTX_SCHED_BLOCKING_SYNC); CuPy then retains that same primary context, and the zero-copy encoder (primary_ctx=True) can register CuPy's device pointers.

Returns True on success. Returns False (and changes nothing) if the CUDA driver can't be loaded. If the primary context is already active with different flags (e.g. CuPy already ran), the driver call may still succeed but the flags will not take effect until the context is reset — so order matters.

Parameters:

Name Type Description Default
device_id int

CUDA device ordinal.

 0
sched str

One of "blocking_sync" (default, required for FFmpeg sharing), "spin", "yield", "auto".

 _FFMPEG_SCHED
Source code in src/pdum/rfb/gpu.py 
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
def enable_cuda_context_sharing(device_id: int = 0, *, sched: str = _FFMPEG_SCHED) -> bool:
    """Pre-set the device primary-context flags so CuPy and FFmpeg share one context.

    **Call this once, first thing in your program, before any CuPy/PyTorch CUDA
    op.** It sets the device's primary-context scheduling flags to what FFmpeg's
    CUDA hwcontext wants (``CU_CTX_SCHED_BLOCKING_SYNC``); CuPy then retains that
    same primary context, and the zero-copy encoder (``primary_ctx=True``) can
    register CuPy's device pointers.

    Returns ``True`` on success. Returns ``False`` (and changes nothing) if the
    CUDA driver can't be loaded. If the primary context is *already active* with
    different flags (e.g. CuPy already ran), the driver call may still succeed but
    the flags will not take effect until the context is reset — so order matters.

    Parameters
    ----------
    device_id:
        CUDA device ordinal.
    sched:
        One of ``"blocking_sync"`` (default, required for FFmpeg sharing),
        ``"spin"``, ``"yield"``, ``"auto"``.
    """
    cu = _libcuda()
    if cu is None:
        return False
    flag = _SCHED_FLAGS[sched]
    if cu.cuInit(0) != 0:
        return False
    dev = ctypes.c_int()
    if cu.cuDeviceGet(ctypes.byref(dev), device_id) != 0:
        return False
    return cu.cuDevicePrimaryCtxSetFlags(dev, flag) == 0

nv12_height(packed)

Image height encoded by a contiguous NV12 (H + H//2, W) buffer.

Source code in src/pdum/rfb/gpu.py 
206
207
208
209
def nv12_height(packed: Any) -> int:
    """Image height encoded by a contiguous NV12 ``(H + H//2, W)`` buffer."""
    rows = int(_as_cupy(packed).shape[0])
    return rows * 2 // 3

nv12_planes(packed)

Slice a contiguous NV12 (H + H//2, W) buffer into (Y, UV) planes.

The two returned CuPy arrays are views into packed (no copy), suitable for av.VideoFrame.from_dlpack([y, uv], format="nv12", ...).

Source code in src/pdum/rfb/gpu.py 
192
193
194
195
196
197
198
199
200
201
202
203
def nv12_planes(packed: Any) -> tuple[Any, Any]:
    """Slice a contiguous NV12 ``(H + H//2, W)`` buffer into ``(Y, UV)`` planes.

    The two returned CuPy arrays are *views* into ``packed`` (no copy), suitable
    for ``av.VideoFrame.from_dlpack([y, uv], format="nv12", ...)``.
    """
    packed = _as_cupy(packed)
    rows = int(packed.shape[0])
    h = rows * 2 // 3
    if h + h // 2 != rows:
        raise ValueError(f"not an NV12 buffer: {rows} rows is not 3/2 of an even height")
    return packed[:h], packed[h:]

rgb_to_nv12(rgb, *, out=None)

Convert a device rgb24 tensor (H, W, 3) to contiguous NV12.

Returns a CuPy uint8 array of shape (H + H//2, W): the Y plane (H rows) immediately followed by the interleaved UV plane (H//2 rows) — the single-allocation layout NVENC requires. Pass out to reuse a buffer across frames (the zero-copy encoder does this).

Source code in src/pdum/rfb/gpu.py 
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def rgb_to_nv12(rgb: Any, *, out: Any | None = None):
    """Convert a device ``rgb24`` tensor ``(H, W, 3)`` to contiguous NV12.

    Returns a CuPy ``uint8`` array of shape ``(H + H//2, W)``: the Y plane
    (``H`` rows) immediately followed by the interleaved UV plane (``H//2``
    rows) — the single-allocation layout NVENC requires. Pass ``out`` to reuse a
    buffer across frames (the zero-copy encoder does this).
    """
    import cupy as cp

    rgb = _as_cupy(rgb)
    if rgb.ndim != 3 or rgb.shape[2] < 3:
        raise ValueError(f"rgb_to_nv12 expects (H, W, 3); got shape {rgb.shape!r}")
    h, w = int(rgb.shape[0]), int(rgb.shape[1])
    if w % 2 or h % 2:
        raise ValueError(f"NV12 requires even dimensions; got {w}x{h}")
    if rgb.shape[2] != 3 or not rgb.flags.c_contiguous:
        rgb = cp.ascontiguousarray(rgb[:, :, :3])
    if out is None:
        out = cp.empty((h + h // 2, w), cp.uint8)
    k_rgb2nv12, _, block = _kernels()
    k_rgb2nv12(_grid(w, h, block), block, (rgb, out, w, h))
    return out

to_host_rgb(frame)

Download a CUDA frame to a contiguous host numpy rgb24 array.

Used by the image / CPU H.264 encoders so an image-only (or CPU-fallback) client still works when the publisher pushes CUDA frames. rgba8 drops alpha; nv12 is converted to rgb24 on the GPU first. A host frame is returned (coerced to 3-channel) unchanged.

Source code in src/pdum/rfb/gpu.py 
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
def to_host_rgb(frame: RawFrame):
    """Download a CUDA frame to a contiguous host ``numpy`` **rgb24** array.

    Used by the image / CPU H.264 encoders so an image-only (or CPU-fallback)
    client still works when the publisher pushes CUDA frames. ``rgba8`` drops
    alpha; ``nv12`` is converted to ``rgb24`` on the GPU first. A host frame is
    returned (coerced to 3-channel) unchanged.
    """
    import numpy as np

    if frame.memory == "cpu":
        arr = frame.data
        return np.ascontiguousarray(arr[:, :, :3]) if getattr(arr, "ndim", 0) == 3 and arr.shape[2] == 4 else arr

    import cupy as cp

    arr = _as_cupy(frame.data)
    if frame.pixel_format in ("rgb24", "rgba8"):
        return cp.asnumpy(cp.ascontiguousarray(arr[:, :, :3]) if arr.shape[2] == 4 else arr)
    if frame.pixel_format == "nv12":
        w, h = frame.width, frame.height
        rgb = cp.empty((h, w, 3), cp.uint8)
        _, k_nv122rgb, block = _kernels()
        k_nv122rgb(_grid(w, h, block), block, (cp.ascontiguousarray(arr), rgb, w, h))
        return cp.asnumpy(rgb)
    raise ValueError(f"cannot convert {frame.pixel_format!r} CUDA frame to host rgb")

metal

Apple Metal / MLX GPU frame helpers — the unified-memory analog of :mod:pdum.rfb.gpu.

Where :mod:pdum.rfb.gpu lets a CUDA tensor be encoded by NVENC without a host round-trip, this module lets an MLX (Apple Metal, unified-memory) frame be converted RGB(A) → NV12 on the GPU with a custom mx.fast.metal_kernel and handed to the VideoToolbox encoder — instead of the CPU color-conversion pass (~6.6 ms at 1080p, and it pegs a core). On Apple Silicon there is no PCIe upload to eliminate, so the remaining copy (host NV12 → CVPixelBuffer) is negligible (≤2 % of frame time, measured); the win here is moving the color conversion onto the GPU.

Everything lazy-imports mlx so import pdum.rfb never requires it. macOS + MLX only; gate on :func:mlx_available. The natural producer is a render kernel that writes an (H, W, 4) RGBA mx.array (see examples/mlx_vt_stream.py); publish it directly (display.publish(rgba) — an MLX array is recognized as a memory="metal" frame) or wrap a pre-converted NV12 array with :func:metal_frame.

MetalHostFrameAdapter

Wrap a host :class:~pdum.rfb.types.EncoderBackend so it tolerates Metal frames.

The Metal analog of :class:pdum.rfb.gpu.HostFrameAdapter: when the publisher pushes MLX (Metal) frames under serve(gpu=True) on macOS, an image-transport viewer's encoder is wrapped in this adapter, which downloads each Metal frame to host rgb24 (:func:to_host_rgb) before delegating. Host frames pass through untouched.

Source code in src/pdum/rfb/metal.py 
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
class MetalHostFrameAdapter:
    """Wrap a host :class:`~pdum.rfb.types.EncoderBackend` so it tolerates Metal frames.

    The Metal analog of :class:`pdum.rfb.gpu.HostFrameAdapter`: when the publisher pushes MLX
    (Metal) frames under ``serve(gpu=True)`` on macOS, an image-transport viewer's encoder is
    wrapped in this adapter, which downloads each Metal frame to host ``rgb24``
    (:func:`to_host_rgb`) before delegating. Host frames pass through untouched.
    """

    __slots__ = ("_inner",)

    def __init__(self, inner: Any) -> None:
        self._inner = inner

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False):
        return self._inner.encode(self._to_host(frame), force_keyframe=force_keyframe)

    def encode_still(self, frame: RawFrame):
        return self._inner.encode_still(self._to_host(frame))

    @staticmethod
    def _to_host(frame: RawFrame) -> RawFrame:
        return to_host_frame(frame)

    def flush(self):
        return self._inner.flush()

    def close(self) -> None:
        self._inner.close()

materialize(array)

Force a lazy MLX array to compute on the calling thread. MLX binds a lazy graph's nodes to the thread's default stream, so an array built on the publish/loop thread cannot be evaluated on the session's encode worker thread ("no Stream(gpu, 0) in current thread"). :meth:Display.publish calls this for Metal frames so the render is materialized on the loop thread; the (cheap) NV12 conversion then runs safely on the worker thread over the eager buffer.

Source code in src/pdum/rfb/metal.py 
 99
100
101
102
103
104
105
106
107
def materialize(array: Any) -> None:
    """Force a lazy MLX array to compute **on the calling thread**. MLX binds a lazy graph's
    nodes to the thread's default stream, so an array built on the publish/loop thread cannot be
    evaluated on the session's encode *worker* thread ("no Stream(gpu, 0) in current thread").
    :meth:`Display.publish` calls this for Metal frames so the render is materialized on the loop
    thread; the (cheap) NV12 conversion then runs safely on the worker thread over the eager buffer."""
    import mlx.core as mx

    mx.eval(array)

metal_frame(array, *, pixel_format='auto', width=None, height=None, seq=0, timestamp_us=0)

Wrap an MLX (Metal) array as a memory="metal" :class:~pdum.rfb.types.RawFrame for publish() — the Metal analog of :func:pdum.rfb.gpu.cuda_frame.

pixel_format="auto" infers from shape: (H, W, 3) → rgb24, (H, W, 4) → rgba8, 2-D (H+H//2, W) → nv12. Use this for a pre-converted NV12 array; a plain RGBA array can be handed straight to display.publish() (it is recognized as a Metal frame). The array is referenced, not copied; keep it alive (and evaluated) until it is encoded.

Source code in src/pdum/rfb/metal.py 
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def metal_frame(
    array: Any,
    *,
    pixel_format: str = "auto",
    width: int | None = None,
    height: int | None = None,
    seq: int = 0,
    timestamp_us: int = 0,
) -> RawFrame:
    """Wrap an MLX (Metal) array as a ``memory="metal"`` :class:`~pdum.rfb.types.RawFrame` for
    ``publish()`` — the Metal analog of :func:`pdum.rfb.gpu.cuda_frame`.

    ``pixel_format="auto"`` infers from shape: ``(H, W, 3)`` → ``rgb24``, ``(H, W, 4)`` → ``rgba8``,
    2-D ``(H+H//2, W)`` → ``nv12``. Use this for a **pre-converted NV12** array; a plain RGBA array
    can be handed straight to ``display.publish()`` (it is recognized as a Metal frame). The array
    is referenced, not copied; keep it alive (and evaluated) until it is encoded.
    """
    import mlx.core as mx

    if not isinstance(array, mx.array):
        array = mx.array(array)
    shape = tuple(int(s) for s in array.shape)
    if pixel_format == "auto":
        if len(shape) == 3 and shape[2] == 3:
            pixel_format = "rgb24"
        elif len(shape) == 3 and shape[2] == 4:
            pixel_format = "rgba8"
        elif len(shape) == 2:
            pixel_format = "nv12"
        else:
            raise ValueError(f"cannot infer pixel_format from shape {shape!r}")
    if pixel_format == "nv12":
        width = shape[1] if width is None else width
        height = (shape[0] * 2 // 3) if height is None else height
    else:
        height = shape[0] if height is None else height
        width = shape[1] if width is None else width
    return RawFrame(
        seq=seq,
        width=int(width),
        height=int(height),
        timestamp_us=int(timestamp_us),
        pixel_format=pixel_format,  # type: ignore[arg-type]
        memory="metal",
        data=array,
    )

mlx_available() cached

True if MLX (Apple Metal) is usable in this process (cached). macOS + mlx importable.

Source code in src/pdum/rfb/metal.py 
30
31
32
33
34
35
36
37
38
39
40
@functools.lru_cache(maxsize=1)
def mlx_available() -> bool:
    """True if MLX (Apple Metal) is usable in this process (cached). macOS + ``mlx`` importable."""
    if sys.platform != "darwin" or importlib.util.find_spec("mlx") is None:
        return False
    try:
        import mlx.core as mx

        return "gpu" in str(mx.default_device()).lower()
    except Exception:
        return False

rgb_to_nv12(rgb)

Convert a Metal rgb24/rgba8 MLX array (H, W, 3|4) to a contiguous NV12 MLX array (H + H//2, W) on the GPU. Returns a lazy mx.array (call :func:to_host_nv12 or mx.eval to materialize). Even dimensions required.

Source code in src/pdum/rfb/metal.py 
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
def rgb_to_nv12(rgb: Any):
    """Convert a Metal ``rgb24``/``rgba8`` MLX array ``(H, W, 3|4)`` to a contiguous NV12 MLX
    array ``(H + H//2, W)`` on the GPU. Returns a **lazy** ``mx.array`` (call :func:`to_host_nv12`
    or ``mx.eval`` to materialize). Even dimensions required."""
    import mlx.core as mx

    if not isinstance(rgb, mx.array):
        rgb = mx.array(rgb)
    if rgb.ndim != 3 or rgb.shape[2] < 3:
        raise ValueError(f"rgb_to_nv12 expects (H, W, 3|4); got shape {tuple(rgb.shape)!r}")
    h, w, c = int(rgb.shape[0]), int(rgb.shape[1]), int(rgb.shape[2])
    if w % 2 or h % 2:
        raise ValueError(f"NV12 requires even dimensions; got {w}x{h}")
    (out,) = _nv12_kernel()(
        inputs=[rgb],
        template=[("W", w), ("H", h), ("C", c)],
        grid=(w, h, 1),
        threadgroup=(16, 16, 1),
        output_shapes=[(h + h // 2, w)],
        output_dtypes=[mx.uint8],
    )
    return out

to_host_frame(frame)

Return frame with its data on the host: a Metal frame is downloaded to a host rgb24 :class:~pdum.rfb.types.RawFrame (:func:to_host_rgb); any other frame is returned unchanged. Lets the CPU / image encoders accept a published MLX frame — the frame was materialized on the publish thread, so this cross-thread read is safe.

Source code in src/pdum/rfb/metal.py 
205
206
207
208
209
210
211
212
213
214
def to_host_frame(frame: RawFrame) -> RawFrame:
    """Return ``frame`` with its data on the host: a Metal frame is downloaded to a host
    ``rgb24`` :class:`~pdum.rfb.types.RawFrame` (:func:`to_host_rgb`); any other frame is
    returned unchanged. Lets the CPU / image encoders accept a published MLX frame — the frame
    was materialized on the publish thread, so this cross-thread read is safe."""
    if frame.memory != "metal":
        return frame
    import dataclasses

    return dataclasses.replace(frame, data=to_host_rgb(frame), memory="cpu", pixel_format="rgb24")

to_host_nv12(array)

Evaluate a Metal NV12 MLX array and return a contiguous host numpy view. Unified memory makes np.asarray a near-zero-copy handoff; the VideoToolbox binding then does the small CVPixelBuffer copy.

Source code in src/pdum/rfb/metal.py 
110
111
112
113
114
115
116
117
118
def to_host_nv12(array: Any):
    """Evaluate a Metal NV12 MLX array and return a contiguous host ``numpy`` view. Unified memory
    makes ``np.asarray`` a near-zero-copy handoff; the VideoToolbox binding then does the small
    ``CVPixelBuffer`` copy."""
    import mlx.core as mx
    import numpy as np

    mx.eval(array)
    return np.ascontiguousarray(np.asarray(array))

to_host_rgb(frame)

Download a Metal frame to a contiguous host numpy rgb24 array — used by the image / CPU encoders so an image-only client still works under serve(gpu=True) on macOS. rgba8 drops alpha; nv12 is converted back to RGB (BT.601 limited) on the host.

Source code in src/pdum/rfb/metal.py 
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
def to_host_rgb(frame: RawFrame):
    """Download a Metal frame to a contiguous host ``numpy`` **rgb24** array — used by the image /
    CPU encoders so an image-only client still works under ``serve(gpu=True)`` on macOS. ``rgba8``
    drops alpha; ``nv12`` is converted back to RGB (BT.601 limited) on the host."""
    import mlx.core as mx
    import numpy as np

    arr = frame.data
    if frame.memory != "metal":  # already host
        h = np.asarray(arr)
        return np.ascontiguousarray(h[:, :, :3]) if getattr(h, "ndim", 0) == 3 and h.shape[2] == 4 else h
    mx.eval(arr)
    host = np.asarray(arr)
    if frame.pixel_format in ("rgb24", "rgba8"):
        return np.ascontiguousarray(host[:, :, :3]) if host.shape[2] == 4 else np.ascontiguousarray(host)
    if frame.pixel_format == "nv12":
        return _nv12_to_rgb_host(host, frame.width, frame.height)
    raise ValueError(f"cannot convert {frame.pixel_format!r} Metal frame to host rgb")

metrics

Per-session performance metrics.

Tracks the quantities the implementation guide lists (section 14): encode time, payload bytes, in-flight depth, round-trip ACK latency (send -> displayed), client decode-queue depth, and derived rates (fps, bitrate). The session feeds these; :meth:SessionMetrics.snapshot returns a plain dict for logging, the GET /metrics side channel, and the adaptive-quality controller.

Latencies use an exponential moving average so the "recent" value reacts quickly without storing history; counts are cumulative. Throughput rates (fps, bitrate) are computed over a short rolling time window (:data:_RATE_WINDOW_S) so they track the current cadence — a lifetime average would crawl toward a new frame rate over minutes and never settle.

SessionMetrics dataclass

Mutable accumulator of one session's performance counters.

Source code in src/pdum/rfb/metrics.py 
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
@dataclass(slots=True)
class SessionMetrics:
    """Mutable accumulator of one session's performance counters."""

    started_at: float = 0.0
    updated_at: float = 0.0

    frames_sent: int = 0
    frames_dropped: int = 0
    frames_acked: int = 0
    keyframes_sent: int = 0
    bytes_sent: int = 0

    encode_ms: float = 0.0  # EMA of encoder.encode() wall time
    rtt_ms: float = 0.0  # EMA of send -> displayed-ack round trip
    decode_queue_size: int = 0  # last value reported by the client

    # Reflected from the session so a snapshot is self-contained.
    inflight: int = 0
    target_bitrate: int = 0
    target_fps: int = 0

    _extra: dict = field(default_factory=dict)
    # Rolling windows of recent event timestamps for windowed rates. `_sent_window`
    # holds (t, payload_bytes) per sent frame; `_acked_window` holds ack timestamps.
    _sent_window: deque[tuple[float, int]] = field(default_factory=deque)
    _acked_window: deque[float] = field(default_factory=deque)

    def record_encode(self, ms: float, *, now: float) -> None:
        self.encode_ms = _ema(self.encode_ms, ms)
        self.updated_at = now

    def record_sent(self, *, payload_bytes: int, keyframe: bool, now: float) -> None:
        self.frames_sent += 1
        self.bytes_sent += payload_bytes
        if keyframe:
            self.keyframes_sent += 1
        self._sent_window.append((now, payload_bytes))
        self.updated_at = now

    def record_dropped(self, *, now: float) -> None:
        self.frames_dropped += 1
        self.updated_at = now

    def record_ack(self, *, rtt_ms: float | None, decode_queue_size: int, now: float) -> None:
        self.frames_acked += 1
        if rtt_ms is not None:
            self.rtt_ms = _ema(self.rtt_ms, rtt_ms)
        self.decode_queue_size = decode_queue_size
        self._acked_window.append(now)
        self.updated_at = now

    def snapshot(self, *, now: float) -> dict:
        """Return a JSON-serializable view including derived rates."""
        elapsed = max(1e-6, now - self.started_at)
        # Windowed throughput: only the last `_RATE_WINDOW_S` of activity. The span
        # is the window unless the session is younger, so early rates aren't diluted.
        cutoff = now - _RATE_WINDOW_S
        while self._sent_window and self._sent_window[0][0] < cutoff:
            self._sent_window.popleft()
        while self._acked_window and self._acked_window[0] < cutoff:
            self._acked_window.popleft()
        span = min(_RATE_WINDOW_S, elapsed)
        window_bytes = sum(b for _, b in self._sent_window)
        return {
            "elapsed_s": round(elapsed, 3),
            "frames_sent": self.frames_sent,
            "frames_dropped": self.frames_dropped,
            "frames_acked": self.frames_acked,
            "keyframes_sent": self.keyframes_sent,
            "bytes_sent": self.bytes_sent,
            "fps_sent": round(len(self._sent_window) / span, 2),
            "fps_acked": round(len(self._acked_window) / span, 2),
            "bitrate_bps": round(window_bytes * 8 / span),
            "encode_ms": round(self.encode_ms, 2),
            "rtt_ms": round(self.rtt_ms, 2),
            "decode_queue_size": self.decode_queue_size,
            "inflight": self.inflight,
            "target_bitrate": self.target_bitrate,
            "target_fps": self.target_fps,
        }

snapshot(*, now)

Return a JSON-serializable view including derived rates.

Source code in src/pdum/rfb/metrics.py 
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
def snapshot(self, *, now: float) -> dict:
    """Return a JSON-serializable view including derived rates."""
    elapsed = max(1e-6, now - self.started_at)
    # Windowed throughput: only the last `_RATE_WINDOW_S` of activity. The span
    # is the window unless the session is younger, so early rates aren't diluted.
    cutoff = now - _RATE_WINDOW_S
    while self._sent_window and self._sent_window[0][0] < cutoff:
        self._sent_window.popleft()
    while self._acked_window and self._acked_window[0] < cutoff:
        self._acked_window.popleft()
    span = min(_RATE_WINDOW_S, elapsed)
    window_bytes = sum(b for _, b in self._sent_window)
    return {
        "elapsed_s": round(elapsed, 3),
        "frames_sent": self.frames_sent,
        "frames_dropped": self.frames_dropped,
        "frames_acked": self.frames_acked,
        "keyframes_sent": self.keyframes_sent,
        "bytes_sent": self.bytes_sent,
        "fps_sent": round(len(self._sent_window) / span, 2),
        "fps_acked": round(len(self._acked_window) / span, 2),
        "bitrate_bps": round(window_bytes * 8 / span),
        "encode_ms": round(self.encode_ms, 2),
        "rtt_ms": round(self.rtt_ms, 2),
        "decode_queue_size": self.decode_queue_size,
        "inflight": self.inflight,
        "target_bitrate": self.target_bitrate,
        "target_fps": self.target_fps,
    }

notebook

anywidget (Jupyter + marimo) front-end for pdum.rfb.

Optional — install the extra::

uv add 'habemus-papadum-rfb[anywidget]'

The widget loads a single self-contained ESM bundle (the Web Worker is inlined) that drives the same RemoteFramebufferView as the standalone browser client. Frames travel over a plain WebSocket, not the Jupyter/ipywidgets kernel comm — so the notebook only carries the url/token traits, never pixels.

One widget = one Web Worker + one WebSocket. The Python Server hub multiplexes many streams on one port, so N cells = N widgets = N independent streams (see docs/notebook.md). Typical use::

import pdum.rfb as rfb
from pdum.rfb.notebook import publish_loop

display = await rfb.serve(1280, 720, port=0)     # top-level await; loop already running
task = publish_loop(display, lambda: render(), fps=30)   # non-blocking background task
display.widget()                                  # -> batteries viewer in the cell

RfbCanvas

Bases: AnyWidget

Bare tier: just the framebuffer canvas filling the cell — you supply the chrome/CSS.

Connection traits (url/host/base_path/port/stream/token/ image_only/main_thread_present) are connect-time: mutating one rebuilds the view. main_thread_present defaults host-aware — True only under marimo (draws on the main thread so the widget survives marimo's reparenting), False elsewhere (the lower-overhead OffscreenCanvas path that also carries the backend-switch/zoom chrome). height sizes the output (a notebook output <div> is 0-height by default, so the canvas would fall back to 320×240 without it). state/stats/last_error are read back from JS.

Source code in src/pdum/rfb/notebook.py 
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
class RfbCanvas(anywidget.AnyWidget):
    """Bare tier: just the framebuffer canvas filling the cell — you supply the chrome/CSS.

    Connection traits (``url``/``host``/``base_path``/``port``/``stream``/``token``/
    ``image_only``/``main_thread_present``) are connect-time: mutating one rebuilds the view.
    ``main_thread_present`` defaults host-aware — ``True`` only under marimo (draws on the main
    thread so the widget survives marimo's reparenting), ``False`` elsewhere (the lower-overhead
    OffscreenCanvas path that also carries the backend-switch/zoom chrome). ``height`` sizes the
    output (a notebook output ``<div>`` is 0-height by default, so the canvas would fall
    back to 320×240 without it). ``state``/``stats``/``last_error`` are read back from JS.
    """

    _esm = _ESM
    _css = _CSS

    # --- connection (connect-time) ---
    url = traitlets.Unicode("").tag(sync=True)  # explicit override; wins over host/port
    host = traitlets.Unicode("auto").tag(sync=True)  # "auto" -> the browser's location.hostname
    base_path = traitlets.Unicode("").tag(sync=True)  # set for same-origin (remote/HTTPS) wss
    port = traitlets.Int(0).tag(sync=True)
    stream = traitlets.Unicode("default").tag(sync=True)
    token = traitlets.Unicode("").tag(sync=True)
    image_only = traitlets.Bool(False).tag(sync=True)
    # Present path. True: draw each frame on the MAIN THREAD (a normal <canvas> the framework may
    # freely reparent) — robust under marimo, which reparents the widget subtree and blanks the
    # OffscreenCanvas-transfer path. False: transfer an OffscreenCanvas to the worker (lower
    # overhead, and the path that carries the backend-switch/zoom chrome). The default is
    # host-aware — True only under marimo, False everywhere else (Jupyter/JupyterLab don't
    # reparent) — so only marimo pays the cost; override explicitly for any other reparenting host.
    main_thread_present = traitlets.Bool().tag(sync=True)

    @traitlets.default("main_thread_present")
    def _default_main_thread_present(self) -> bool:
        return _running_in_marimo()

    # Fit mode when the frame AR differs from the canvas AR ("contain" | "cover" | "fill";
    # default "contain" client-side); background is the letterbox fill for "contain".
    fit = traitlets.Unicode("").tag(sync=True)
    background = traitlets.Unicode("").tag(sync=True)
    height = traitlets.Int(480).tag(sync=True)

    # --- chrome (off in the bare tier) ---
    show_toolbar = traitlets.Bool(False).tag(sync=True)
    show_stats = traitlets.Bool(False).tag(sync=True)

    # --- readback (JS -> Python; observable) ---
    state = traitlets.Unicode("connecting").tag(sync=True)
    stats = traitlets.Dict().tag(sync=True)
    last_error = traitlets.Unicode("").tag(sync=True)

RfbViewer

Bases: RfbCanvas

Batteries tier: status pill + latency badge + toggleable stats HUD + toolbar.

Same connection traits as :class:RfbCanvas, with the chrome on by default. Theme via the CSS custom properties on .rfb-root (a cell-injected <style> or a JupyterLab theme); drop chrome per-widget with show_toolbar=False / show_stats=False.

Source code in src/pdum/rfb/notebook.py 
167
168
169
170
171
172
173
174
175
176
class RfbViewer(RfbCanvas):
    """Batteries tier: status pill + latency badge + toggleable stats HUD + toolbar.

    Same connection traits as :class:`RfbCanvas`, with the chrome on by default. Theme via
    the CSS custom properties on ``.rfb-root`` (a cell-injected ``<style>`` or a JupyterLab
    theme); drop chrome per-widget with ``show_toolbar=False`` / ``show_stats=False``.
    """

    show_toolbar = traitlets.Bool(True).tag(sync=True)
    show_stats = traitlets.Bool(True).tag(sync=True)

publish_loop(display, render, *, fps=30)

Schedule render() → display.publish() as a background task; return it immediately.

Non-blocking, so a notebook cell keeps going while frames flow (Jupyter/marimo already run an asyncio loop, and await rfb.serve(...) works with top-level await). Tear down with task.cancel() then await display.aclose(). To handle input, poll display.poll_events() from inside render or run your own loop instead; the event queue is bounded, so leaving it unpolled is safe.

Source code in src/pdum/rfb/notebook.py 
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
def publish_loop(display: "Display", render: Callable[[], Any], *, fps: int = 30) -> "asyncio.Task[None]":
    """Schedule ``render()`` → ``display.publish()`` as a background task; return it immediately.

    Non-blocking, so a notebook cell keeps going while frames flow (Jupyter/marimo already
    run an asyncio loop, and ``await rfb.serve(...)`` works with top-level await). Tear down
    with ``task.cancel()`` then ``await display.aclose()``. To handle input, poll
    ``display.poll_events()`` from inside ``render`` or run your own loop instead; the event
    queue is bounded, so leaving it unpolled is safe.
    """
    period = 1.0 / fps

    async def _run() -> None:
        try:
            while not display._closed:
                display.publish(render())
                await asyncio.sleep(period)
        except asyncio.CancelledError:  # graceful stop on task.cancel()
            pass

    return asyncio.ensure_future(_run())

protocol

Wire protocol: binary envelope, header builders, and capability negotiation.

The transport is transport-neutral JSON for control plus a simple binary envelope for image/video payloads::

uint32le header_byte_length
utf8 JSON header
raw payload bytes

These functions are pure (no I/O) so they are fully unit-testable and the wire shape is defined in exactly one place, shared by the session and the tests. The binary envelope must stay byte-for-byte compatible with the JavaScript unpackBinaryMessage in widgets/src/protocol.ts.

BackendSelection dataclass

The encoder/transport the server chose for a connection.

Source code in src/pdum/rfb/protocol.py 
180
181
182
183
184
185
186
187
@dataclass(slots=True)
class BackendSelection:
    """The encoder/transport the server chose for a connection."""

    transport: Literal["image", "h264"]
    mime: str | None = None  # for the image transport
    codec: str | None = None  # for the h264 transport, e.g. "avc1.42E01F"
    image_mode: ImageMode | None = None

UnsupportedClient

Bases: Exception

Raised when a client advertises no transport the server can satisfy.

Source code in src/pdum/rfb/protocol.py 
43
44
class UnsupportedClient(Exception):
    """Raised when a client advertises no transport the server can satisfy."""

config_message(*, transport, width, height, codec=None, pixel_ratio=1.0, color=None, coords='frame-pixels')

Build the server config control message (sent right after hello).

coords declares the event coordinate space the client sends (always "frame-pixels" in this version — pointer/wheel x/y index the published framebuffer directly). pixel_ratio is the frame's render DPR and color an optional color descriptor (the dict form of a :class:~pdum.rfb.types.ColorSpace); both are omitted when at their defaults so older clients are unaffected.

Source code in src/pdum/rfb/protocol.py 
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
def config_message(
    *,
    transport: str,
    width: int,
    height: int,
    codec: str | None = None,
    pixel_ratio: float = 1.0,
    color: dict | None = None,
    coords: str = "frame-pixels",
) -> str:
    """Build the server ``config`` control message (sent right after ``hello``).

    ``coords`` declares the event coordinate space the client sends (always
    ``"frame-pixels"`` in this version — pointer/wheel ``x``/``y`` index the published
    framebuffer directly). ``pixel_ratio`` is the frame's render DPR and ``color`` an
    optional color descriptor (the ``dict`` form of a
    :class:`~pdum.rfb.types.ColorSpace`); both are omitted when at their defaults so
    older clients are unaffected.
    """
    msg: dict = {
        "type": "config",
        "transport": transport,
        "width": width,
        "height": height,
        "coords": coords,
    }
    if codec is not None:
        msg["codec"] = codec
    if pixel_ratio is not None and pixel_ratio != 1.0:
        msg["pixel_ratio"] = pixel_ratio
    if color is not None:
        msg["color"] = color
    return json.dumps(msg, separators=(",", ":"))

header_for(p)

Return the appropriate binary-envelope header for p.

Source code in src/pdum/rfb/protocol.py 
121
122
123
def header_for(p: EncodedPayload) -> dict:
    """Return the appropriate binary-envelope header for ``p``."""
    return image_header(p) if p.kind == "image" else video_header(p)

image_header(p)

Build the binary-envelope header for an image frame.

Source code in src/pdum/rfb/protocol.py 
87
88
89
90
91
92
93
94
95
96
97
def image_header(p: EncodedPayload) -> dict:
    """Build the binary-envelope header for an image frame."""
    return {
        "type": "image_frame",
        "seq": p.seq,
        "timestamp_us": p.timestamp_us,
        "width": p.width,
        "height": p.height,
        "mime": p.mime,
        **_render_descriptors(p),
    }

pack_binary_message(header, payload)

Pack a header dict and payload bytes into a single binary message.

The header is encoded as compact UTF-8 JSON (no spaces) prefixed by its little-endian uint32 byte length.

Source code in src/pdum/rfb/protocol.py 
47
48
49
50
51
52
53
54
def pack_binary_message(header: dict, payload: bytes) -> bytes:
    """Pack a header dict and payload bytes into a single binary message.

    The header is encoded as compact UTF-8 JSON (no spaces) prefixed by its
    little-endian ``uint32`` byte length.
    """
    header_bytes = json.dumps(header, separators=(",", ":")).encode("utf-8")
    return struct.pack("<I", len(header_bytes)) + header_bytes + bytes(payload)

parse_control(text)

Parse a JSON control message into a dict.

Source code in src/pdum/rfb/protocol.py 
129
130
131
def parse_control(text: str) -> dict:
    """Parse a JSON control message into a dict."""
    return json.loads(text)

select_transport(client_supported, *, has_h264, has_nvenc=False, prefer_video=True, image_mode='jpeg')

Choose the best backend given client capabilities and server encoders.

Policy (guide section 12): if the client supports WebCodecs/H.264, the server prefers video and at least one H.264 encoder is available, pick H.264 (NVENC is preferred over the CPU path when present). Otherwise fall back to the best mutually-supported image format. has_nvenc is accepted now so the NVENC backend can be slotted in later without touching callers.

Raises:

Type Description
UnsupportedClient

If no mutually-supported transport exists.
Source code in src/pdum/rfb/protocol.py 
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
def select_transport(
    client_supported: list[str],
    *,
    has_h264: bool,
    has_nvenc: bool = False,
    prefer_video: bool = True,
    image_mode: ImageMode = "jpeg",
) -> BackendSelection:
    """Choose the best backend given client capabilities and server encoders.

    Policy (guide section 12): if the client supports WebCodecs/H.264, the
    server prefers video and at least one H.264 encoder is available, pick
    H.264 (NVENC is preferred over the CPU path when present). Otherwise fall
    back to the best mutually-supported image format. ``has_nvenc`` is accepted
    now so the NVENC backend can be slotted in later without touching callers.

    Raises
    ------
    UnsupportedClient
        If no mutually-supported transport exists.
    """
    supported = set(client_supported)

    if prefer_video and CAP_H264_ANNEXB in supported and (has_nvenc or has_h264):
        return BackendSelection(transport="h264", codec=DEFAULT_H264_CODEC)

    # Prefer the caller's requested image mode if the client supports it,
    # then fall back to any mutually-supported image format.
    preferred_cap = _MIME_BY_MODE[image_mode]
    ordered_caps = [preferred_cap, CAP_PNG, CAP_JPEG, CAP_WEBP]
    for cap in ordered_caps:
        if cap in supported:
            mode = _MODE_BY_CAP[cap]
            return BackendSelection(transport="image", mime=cap, image_mode=mode)

    raise UnsupportedClient(f"no supported transport in client capabilities: {sorted(supported)}")

stats_message(*, server_queue, dropped)

Build a server stats control message.

Source code in src/pdum/rfb/protocol.py 
169
170
171
172
173
174
def stats_message(*, server_queue: int, dropped: int) -> str:
    """Build a server ``stats`` control message."""
    return json.dumps(
        {"type": "stats", "server_queue": server_queue, "dropped": dropped},
        separators=(",", ":"),
    )

unpack_binary_message(buf)

Inverse of :func:pack_binary_message.

Returns:

Type Description
tuple[dict, bytes]

The decoded JSON header and the raw payload bytes.
Source code in src/pdum/rfb/protocol.py 
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
def unpack_binary_message(buf: bytes | bytearray | memoryview) -> tuple[dict, bytes]:
    """Inverse of :func:`pack_binary_message`.

    Returns
    -------
    tuple[dict, bytes]
        The decoded JSON header and the raw payload bytes.
    """
    mv = memoryview(buf)
    if len(mv) < 4:
        raise ValueError("buffer too small to contain a header length prefix")
    (n,) = struct.unpack("<I", mv[:4])
    if len(mv) < 4 + n:
        raise ValueError(f"buffer truncated: need {4 + n} bytes, have {len(mv)}")
    header = json.loads(bytes(mv[4 : 4 + n]).decode("utf-8"))
    payload = bytes(mv[4 + n :])
    return header, payload

video_header(p)

Build the binary-envelope header for an encoded video access unit.

Source code in src/pdum/rfb/protocol.py 
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
def video_header(p: EncodedPayload) -> dict:
    """Build the binary-envelope header for an encoded video access unit."""
    bitstream = "annexb"
    if p.metadata and "bitstream" in p.metadata:
        bitstream = p.metadata["bitstream"]
    header = {
        "type": "video_chunk",
        "seq": p.seq,
        "timestamp_us": p.timestamp_us,
        "width": p.width,
        "height": p.height,
        "codec": p.codec,
        "bitstream": bitstream,
        "keyframe": p.keyframe,
    }
    if p.duration_us is not None:
        header["duration_us"] = p.duration_us
    header.update(_render_descriptors(p))
    return header

recording

Server-side MP4 recording: tap the published-frame stream and mux to a file.

A convenience on :class:~pdum.rfb.display.Display (:meth:Display.record) that records what you are publishing to a real MP4 file, independent of any connected browser — so it works fully headless (demos, CI artifacts, golden-frame capture).

Design notes (why this is not the wire path):

  • It reuses the H.264 encoder (libx264) but muxes to a real MP4 container. This is the one place AVCC-in-MP4 is correct — the wire invariant "Annex B only, never route H.264 through an mp4 muxer" is about the WebSocket payload; a file recording wants exactly the AVCC-in-MP4 the wire forbids. So this is a separate PyAV encode→mux pipeline (av.open(..., "w") + add_stream("libx264") + container.mux); it never touches the sessions' Annex B bitstream.
  • It honors the real monotonic timestamps that already ride each :class:~pdum.rfb.types.RawFrame, so a variable-cadence publisher (sparse still_after scenes included) records with correct frame timing (VFR): each frame's PTS is its real timestamp relative to the first recorded frame.

PyAV is imported lazily (the [h264] extra), so import pdum.rfb — and even import pdum.rfb.recording — stays dependency-light.

Recording

Handle for an in-progress server-side MP4 recording.

Created (and started) by :meth:~pdum.rfb.display.Display.record. Use it as an async context manager (auto-finalizes on exit) or call :meth:stop explicitly::

async with display.record("out.mp4"):
    ...            # publish frames
# -- or --
rec = display.record("out.mp4")
...
await rec.stop()   # flush the encoder, finalize the container

Attributes:

Name Type Description
frames_written

Count of frames encoded into the file so far (useful for tests / progress).
Source code in src/pdum/rfb/recording.py 
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
class Recording:
    """Handle for an in-progress server-side MP4 recording.

    Created (and started) by :meth:`~pdum.rfb.display.Display.record`. Use it as an async
    context manager (auto-finalizes on exit) or call :meth:`stop` explicitly::

        async with display.record("out.mp4"):
            ...            # publish frames
        # -- or --
        rec = display.record("out.mp4")
        ...
        await rec.stop()   # flush the encoder, finalize the container

    Attributes
    ----------
    frames_written:
        Count of frames encoded into the file so far (useful for tests / progress).
    """

    def __init__(
        self,
        display: Display,
        path: str | Path,
        *,
        fps: int,
        bitrate: int | None = None,
        crf: int | None = None,
        preset: str = "veryfast",
    ) -> None:
        if not recording_available():
            raise RuntimeError(
                "Display.record() needs PyAV (the [h264] extra). Install with pip install 'habemus-papadum-rfb[h264]'."
            )
        if bitrate is not None and crf is not None:
            raise ValueError("pass at most one of bitrate / crf")
        self._display = display
        self.path = Path(path)
        self.fps = int(fps)
        self.bitrate = bitrate
        self.crf = crf
        self.preset = preset
        self.frames_written = 0

        self._tap = _RecordingTap(display)
        self._task: asyncio.Task | None = None
        self._stopped = False
        self._error: BaseException | None = None
        # PyAV objects, created lazily on the first frame (they need its dimensions).
        self._container: Any = None
        self._stream: Any = None
        self._size: tuple[int, int] | None = None
        self._t0_us: int | None = None
        self._last_pts: int = -1

    # --- lifecycle ---------------------------------------------------------

    def _start_task(self) -> None:
        """Launch the background encode task (called by :meth:`Display.record`)."""
        self._task = asyncio.create_task(self._run())

    async def stop(self) -> None:
        """Stop recording and finalize the MP4 file (idempotent).

        Signals the tap to unpark, awaits the encode task (which flushes the encoder and
        closes the container in its ``finally``), then re-raises any error the task hit.
        """
        if self._stopped:
            return
        self._stopped = True
        self._tap.close()
        if self._task is not None:
            await self._task
            self._task = None
        if self._error is not None:
            raise self._error

    async def __aenter__(self) -> Recording:
        return self

    async def __aexit__(self, *exc: Any) -> None:
        await self.stop()

    # --- encode loop -------------------------------------------------------

    async def _run(self) -> None:
        try:
            # Open the container + libx264 context up front (off-thread), sized to the
            # display's current dimensions, so the one-time encoder init doesn't stall the
            # loop or make us drop the first frames while it warms up. Later frames of a
            # different size are scaled to fit in _write_frame.
            await asyncio.to_thread(self._ensure_container, self._display.width, self._display.height)
            while True:
                frame = await self._tap.next_frame()
                if frame is None:
                    break
                # Snapshot the (borrowed) pixels on THIS loop thread — no await between
                # next_frame() and here, so it is atomic w.r.t. a concurrent publish() —
                # then encode + mux off-thread so the event loop keeps serving viewers.
                host, av_format = _snapshot_host(frame)
                ts = int(frame.timestamp_us)
                await asyncio.to_thread(self._write_frame, host, av_format, ts)
        except BaseException as exc:  # noqa: BLE001 - stash and re-raise from stop()
            self._error = exc
        finally:
            self._display._remove_tap(self._tap)
            await asyncio.to_thread(self._finalize)

    def _ensure_container(self, width: int, height: int) -> None:
        """Open the MP4 container + libx264 stream, sized to the first frame."""
        import av

        self._container = av.open(str(self.path), mode="w")
        stream = self._container.add_stream("libx264", rate=self.fps)
        stream.width = width
        stream.height = height
        stream.pix_fmt = "yuv420p"
        stream.codec_context.time_base = _TIME_BASE
        # No B-frames (tune=zerolatency) ⇒ output DTS == PTS, monotonic — trivial to mux
        # with our real-timestamp PTS. Constant-quality by default; bitrate/crf override.
        options = {"preset": self.preset, "tune": "zerolatency"}
        if self.bitrate is not None:
            stream.bit_rate = int(self.bitrate)
        else:
            options["crf"] = str(self.crf if self.crf is not None else 20)
        stream.options = options
        self._stream = stream
        self._size = (width, height)

    def _write_frame(self, host: np.ndarray, av_format: str, ts_us: int) -> None:
        """Encode one host frame and mux it (runs in a worker thread)."""
        import av

        h, w = host.shape[0], host.shape[1]
        if self._container is None:
            self._ensure_container(w, h)
        assert self._size is not None
        vframe = av.VideoFrame.from_ndarray(host, format=av_format)
        # Fix the frame size to the stream's (scale any later resize) and go to yuv420p.
        sw, sh = self._size
        vframe = vframe.reformat(width=sw, height=sh, format="yuv420p")

        if self._t0_us is None:
            self._t0_us = ts_us
        pts = ts_us - self._t0_us
        # PTS must strictly increase (a repeated/settled timestamp would stall the muxer).
        if pts <= self._last_pts:
            pts = self._last_pts + 1
        self._last_pts = pts
        vframe.pts = pts
        vframe.time_base = _TIME_BASE

        for packet in self._stream.encode(vframe):
            self._container.mux(packet)
        self.frames_written += 1

    def _finalize(self) -> None:
        """Flush the encoder and close the container (runs in a worker thread)."""
        if self._container is None:
            return
        try:
            if self._stream is not None:
                for packet in self._stream.encode(None):  # drain buffered packets
                    self._container.mux(packet)
        finally:
            self._container.close()
            self._container = None
            self._stream = None

stop() async

Stop recording and finalize the MP4 file (idempotent).

Signals the tap to unpark, awaits the encode task (which flushes the encoder and closes the container in its finally), then re-raises any error the task hit.

Source code in src/pdum/rfb/recording.py 
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
async def stop(self) -> None:
    """Stop recording and finalize the MP4 file (idempotent).

    Signals the tap to unpark, awaits the encode task (which flushes the encoder and
    closes the container in its ``finally``), then re-raises any error the task hit.
    """
    if self._stopped:
        return
    self._stopped = True
    self._tap.close()
    if self._task is not None:
        await self._task
        self._task = None
    if self._error is not None:
        raise self._error

recording_available()

True if PyAV is importable (the [h264] extra), so recording can run.

Source code in src/pdum/rfb/recording.py 
45
46
47
def recording_available() -> bool:
    """True if PyAV is importable (the ``[h264]`` extra), so recording can run."""
    return importlib.util.find_spec("av") is not None

rendercanvas

A rendercanvas <https://rendercanvas.readthedocs.io>_ backend that streams over pdum.rfb.

rendercanvas is the canvas-abstraction layer under wgpu / pygfx / fastplotlib: a render engine targets an abstract canvas and a backend decides where the pixels go (a glfw window, a Qt widget, an offscreen buffer, a notebook...). This module is a backend whose pixels go to a :class:pdum.rfb.Display — i.e. a wgpu/ pygfx app renders unchanged and the result streams to the browser over this library's WebSocket + Web-Worker pipeline (image or H.264/WebCodecs), with input flowing back.

It is the spiritual equivalent of rendercanvas's own jupyter_rfb backend, but on this library's transport. Cross-platform: the "bitmap" present method downloads the rendered frame to a host numpy array, so it works identically on macOS and Linux (no CUDA/NVENC required). The GPU zero-copy path is a separate, Linux-only future track (see docs/proposals/active/wgpu_nvenc_zerocopy.md).

Usage (own your asyncio loop; reuse :func:pdum.rfb.serve)::

import asyncio, pdum.rfb as rfb
from pdum.rfb.rendercanvas import RfbRenderCanvas, loop
import pygfx

async def main():
    display = await rfb.serve(1280, 720, port=8765)
    canvas = RfbRenderCanvas(display=display, size=(1280, 720))
    renderer = pygfx.renderers.WgpuRenderer(canvas)
    scene, camera = build_scene()                  # your pygfx scene
    controller = pygfx.OrbitController(camera, register_events=renderer)

    def animate():
        renderer.render(scene, camera)
        canvas.request_draw(animate)

    canvas.request_draw(animate)
    try:
        await loop.run_async()                     # runs on the current asyncio loop
    finally:
        await display.aclose()

asyncio.run(main())
Notes
  • Browser input (pointer / wheel / key) is drained from the display and delivered to the canvas event system, so pygfx controllers (orbit camera, etc.) work. When you use this backend the events go to the canvas (canvas.add_event_handler / controllers), not to display.poll_events() — the backend drains that queue for you.
  • The canvas size (set at construction or via set_logical_size) is the render resolution and what gets published. Browser resize is informational here (the publisher owns the resolution), so it is not auto-applied — matching the shared-display model. Keep the size even for the H.264 path.
  • Event-schema note: this library emits the renderview <https://github.com/pygfx/renderview>_ vocabulary (type/timestamp); rendercanvas 2.x still consumes the legacy keys (event_type/time_stamp). :func:_to_rendercanvas_event renames them — the values (logical coords, 1=left/2=right/3=middle buttons, tuple buttons, capitalized modifiers) already match, so it is a pure key-rename. When rendercanvas adopts type it collapses to the identity.

RfbCanvasGroup

Bases: BaseCanvasGroup

Canvas group binding :class:RfbRenderCanvas to the shared asyncio loop.

Source code in src/pdum/rfb/rendercanvas.py 
107
108
class RfbCanvasGroup(BaseCanvasGroup):
    """Canvas group binding :class:`RfbRenderCanvas` to the shared asyncio loop."""

RfbRenderCanvas

Bases: BaseRenderCanvas

A rendercanvas backend that publishes each rendered frame to a :class:~pdum.rfb.display.Display.

Parameters:

Name Type Description Default
display Display

A started :class:~pdum.rfb.display.Display (from await pdum.rfb.serve(...)). Rendered frames are published to it and browser input is drained from it.

 required
size tuple[int, int] | None

Logical canvas size (width, height) — the render resolution and the published frame size. Defaults to the display's current size.

 None
**kwargs Any

Forwarded to :class:rendercanvas.base.BaseRenderCanvas (update_mode, max_fps, title, ...).

 {}
Source code in src/pdum/rfb/rendercanvas.py 
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
class RfbRenderCanvas(BaseRenderCanvas):
    """A ``rendercanvas`` backend that publishes each rendered frame to a :class:`~pdum.rfb.display.Display`.

    Parameters
    ----------
    display:
        A started :class:`~pdum.rfb.display.Display` (from ``await pdum.rfb.serve(...)``).
        Rendered frames are published to it and browser input is drained from it.
    size:
        Logical canvas size ``(width, height)`` — the render resolution and the published
        frame size. Defaults to the display's current size.
    **kwargs:
        Forwarded to :class:`rendercanvas.base.BaseRenderCanvas` (``update_mode``,
        ``max_fps``, ``title``, ...).
    """

    _rc_canvas_group = RfbCanvasGroup(loop)

    def __init__(self, *args: Any, display: Display, size: tuple[int, int] | None = None, **kwargs: Any) -> None:
        self._display = display
        self._closed = False
        if size is None:
            size = (display.width, display.height)
        super().__init__(*args, size=size, **kwargs)
        self._final_canvas_init()

    # --- present (the rendered frame) --------------------------------------

    def _rc_get_present_info(self, present_methods: list[str]) -> dict | None:
        if "bitmap" in present_methods:
            return {"method": "bitmap", "formats": ["rgba-u8"]}
        return None  # we have no native surface, so "screen" is unsupported

    def _rc_present_bitmap(self, *, data: Any, format: str, **kwargs: Any) -> None:
        # `data` is a contiguous (H, W, 4) uint8 RGBA array (wgpu downloaded it from the
        # render texture). publish() tags (H, W, 4) as rgba8 and fans it out to viewers.
        if self._closed or self._display._closed:
            return
        self._display.publish(np.asarray(data))

    # --- scheduling (mirror the loop-driven glfw / offscreen backends) ------

    def _rc_request_draw(self) -> None:
        self._time_to_draw()

    def _rc_request_paint(self) -> None:
        # No native surface to repaint: the frame is already published in _rc_present_bitmap.
        pass

    def _rc_force_paint(self) -> None:
        self._time_to_paint()

    def _rc_gui_poll(self) -> None:
        # Called regularly by the scheduler. Drain browser input from all viewers and
        # deliver it to the canvas event system (pygfx controllers, handlers).
        for ev in self._display.poll_events():
            # poll_events() may also yield a DownscaleHint (adaptive resolution); the
            # backend owns its own sizing, so only input events are relevant here.
            if not isinstance(ev, InputEvent):
                continue
            rc_event = _to_rendercanvas_event(ev.event)
            if rc_event is not None:
                self.submit_event(rc_event)

    # --- size / lifecycle --------------------------------------------------

    def _rc_set_logical_size(self, width: float, height: float) -> None:
        # Render at the logical size 1:1 (ratio 1.0); this is what wgpu uses for the render
        # target and what we publish. A different size simply resizes the display on publish.
        w = max(1, int(width))
        h = max(1, int(height))
        self._size_info.set_physical_size(w, h, 1.0)

    def _rc_close(self) -> None:
        self._closed = True

    def _rc_get_closed(self) -> bool:
        return self._closed

    def _rc_set_title(self, title: str) -> None:
        pass

    def _rc_set_cursor(self, cursor: str) -> None:
        pass

server

WebSocket server + demo CLI for the push-model remote framebuffer.

:func:serve starts a websockets server in the background and returns a live :class:~pdum.rfb.display.Display. The application owns its loop and pushes frames with :meth:Display.publish; each connecting browser is negotiated a transport and driven by its own :class:~pdum.rfb.session.RfbSession, all fed from the display's latest frame.

The same port answers a small HTTP side channel used by the headless e2e harness:

  • GET /health -> ok (readiness probe for Playwright's webServer)
  • GET /recorded-events -> JSON list of every input event received
  • GET /recorded-events/reset -> clears the list (per-test isolation)
  • GET /metrics -> JSON array, one object per active session

python -m pdum.rfb.server is a self-contained demo: it owns a publish loop streaming a deterministic pattern so a browser (or Playwright) can connect.

Server

A hub: one WebSocket listener fronting several named streams.

Each stream is an independent :class:~pdum.rfb.display.Display with its own encoder config; a browser selects one by URL path (ws://host/<stream>), and a connection with no path lands on the "default" stream. Streams are discoverable over HTTP at GET /streams.

Build one with :func:serve_server, or use :func:serve for the common single-default-stream case (it returns the default Display and keeps display.server pointing back here so you can add_stream more).

This composes with everything else — multi-client fan-out, per-client backpressure, the encoders, auth, "still after settle" — none of which changes; a stream is just a Display plus its config, and routing is purely additive.

Source code in src/pdum/rfb/server.py 
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
class Server:
    """A hub: one WebSocket listener fronting several named streams.

    Each stream is an independent :class:`~pdum.rfb.display.Display` with its own
    encoder config; a browser selects one by **URL path** (``ws://host/<stream>``),
    and a connection with no path lands on the ``"default"`` stream. Streams are
    discoverable over HTTP at ``GET /streams``.

    Build one with :func:`serve_server`, or use :func:`serve` for the common
    single-default-stream case (it returns the default ``Display`` and keeps
    ``display.server`` pointing back here so you can ``add_stream`` more).

    This composes with everything else — multi-client fan-out, per-client
    backpressure, the encoders, auth, "still after settle" — none of which changes;
    a stream is just a ``Display`` plus its config, and routing is purely additive.
    """

    def __init__(
        self,
        *,
        host: str = "127.0.0.1",
        port: int = 8765,
        origins: list[str | None] | None = None,
    ) -> None:
        self.host = host
        self._port = port  # requested; the bound port may differ when port=0
        self.origins = origins
        self._streams: dict[str, _StreamHost] = {}
        self._listener: Any = None
        self._listener_cm: Any = None
        self._closed = False

    # --- streams -----------------------------------------------------------

    def add_stream(
        self,
        name: str,
        width: int,
        height: int,
        *,
        fps: int = 30,
        bitrate: int = 12_000_000,
        max_inflight: int = 2,
        has_h264: bool | None = None,
        has_nvenc: bool | None = None,
        gpu: bool = False,
        adaptive: bool = False,
        still_after: float | None = None,
        stats_interval: float | None = None,
        authenticate: Authenticator | None = None,
        record_events: bool = False,
        event_log: str | Path | None = None,
        event_queue_size: int = 4096,
        own_frames: bool = False,
        resize_policy: str = "publisher",
        max_render_dimension: int | None = None,
        encode_pipeline_depth: int = 0,
    ) -> Display:
        """Register a new named stream and return its :class:`Display`.

        Streams are independent: each carries its own encoder config (one GPU, one
        image; per-stream bitrate; per-stream ``authenticate``). Safe to call before
        or after :meth:`start` — clients reach it at ``ws://host/<name>`` either way.
        Raises if ``name`` is already taken. ``own_frames`` / ``resize_policy`` /
        ``max_render_dimension`` are forwarded to the stream's :class:`Display`.
        """
        if name in self._streams:
            raise ValueError(f"stream {name!r} already exists")
        display = Display(
            width,
            height,
            fps=fps,
            record_events=record_events,
            event_log=event_log,
            event_queue_size=event_queue_size,
            own_frames=own_frames,
            resize_policy=resize_policy,
            max_render_dimension=max_render_dimension,
        )
        host = _StreamHost(
            display,
            name,
            has_h264=has_h264,
            has_nvenc=has_nvenc,
            fps=fps,
            bitrate=bitrate,
            max_inflight=max_inflight,
            adaptive=adaptive,
            still_after=still_after,
            stats_interval=stats_interval,
            authenticate=authenticate,
            gpu=gpu,
            encode_pipeline_depth=encode_pipeline_depth,
        )
        self._streams[name] = host
        display._owner_server = self
        display._stream_name = name
        display._server = self._listener  # None until start(); back-filled there
        return display

    def stream(self, name: str = DEFAULT_STREAM) -> Display:
        """Return the :class:`Display` for stream ``name`` (``KeyError`` if absent)."""
        return self._streams[name].display

    def remove_stream(self, name: str) -> None:
        """Remove a named stream, disconnecting its viewers — the inverse of :meth:`add_stream`.

        No-op if ``name`` is absent. The stream's :class:`Display` is closed locally
        (viewers dropped, waiters woken) but the shared listener keeps running for the
        other streams; a later connection to ``name`` closes with ``4404``. Used by the
        demo hub to reap idle per-client streams.
        """
        host = self._streams.pop(name, None)
        if host is not None:
            host.display._close_local()

    @property
    def streams(self) -> list[str]:
        """The names of the registered streams."""
        return list(self._streams)

    @property
    def port(self) -> int | None:
        """The bound TCP port (the actual one when started with ``port=0``)."""
        if self._listener is not None:
            return next(iter(self._listener.sockets)).getsockname()[1]
        return self._port

    # --- lifecycle ---------------------------------------------------------

    async def start(self) -> Server:
        """Start the shared listener in the background; returns ``self``."""
        import websockets.asyncio.server

        kwargs: dict[str, Any] = dict(process_request=self.process_request, max_size=None)
        if self.origins is not None:
            kwargs["origins"] = self.origins
        cm = websockets.asyncio.server.serve(self._route, self.host, self._port, **kwargs)
        self._listener = await cm.__aenter__()
        self._listener_cm = cm
        for host in self._streams.values():
            host.display._server = self._listener
        return self

    async def aclose(self) -> None:
        """Stop the listener and disconnect every viewer of every stream."""
        if self._closed:
            return
        self._closed = True
        if self._listener_cm is not None:
            cm, self._listener_cm = self._listener_cm, None
            self._listener = None
            await cm.__aexit__(None, None, None)
        for host in self._streams.values():
            host.display._close_local()

    async def __aenter__(self) -> Server:
        return await self.start()

    async def __aexit__(self, *exc: Any) -> None:
        await self.aclose()

    # --- routing -----------------------------------------------------------

    @staticmethod
    def _stream_name(path: str) -> str:
        """Map a request path to a stream name (first path segment; ``"default"``)."""
        seg = path.split("?", 1)[0].strip("/").split("/", 1)[0]
        return seg or DEFAULT_STREAM

    async def _route(self, connection: Any) -> None:
        """Dispatch one connection to its stream by URL path (close 4404 if unknown)."""
        req = getattr(connection, "request", None)
        name = self._stream_name(getattr(req, "path", "") or "")
        host = self._streams.get(name)
        if host is None:
            await connection.close(4404, f"unknown stream {name!r}")
            return
        await host.handler(connection)

    def process_request(self, connection: Any, request: Any):
        """Answer the HTTP side-channel routes; return None to proceed with WS.

        Global: ``GET /health``, ``GET /streams``, ``GET /streams/<name>/metrics``.
        For backward compatibility the single-stream routes (``/metrics``,
        ``/recorded-events``, ``/recorded-events/reset``) act on the ``"default"``
        stream when one exists.
        """
        path = request.path.split("?", 1)[0]
        if path == "/health":
            return connection.respond(HTTPStatus.OK, "ok\n")
        if path == "/streams":
            return connection.respond(HTTPStatus.OK, json.dumps([h.info() for h in self._streams.values()]))
        parts = path.strip("/").split("/")
        if len(parts) == 3 and parts[0] == "streams" and parts[2] == "metrics":
            host = self._streams.get(parts[1])
            if host is None:
                return connection.respond(HTTPStatus.NOT_FOUND, "[]")
            return connection.respond(HTTPStatus.OK, json.dumps(host.metrics()))
        default = self._streams.get(DEFAULT_STREAM)
        if default is not None:
            if path == "/metrics":
                return connection.respond(HTTPStatus.OK, json.dumps(default.metrics()))
            if path == "/recorded-events":
                return connection.respond(HTTPStatus.OK, json.dumps(default.display.recorded))
            if path == "/recorded-events/reset":
                default.display.recorded.clear()
                return connection.respond(HTTPStatus.OK, "[]")
        return None

port property

The bound TCP port (the actual one when started with port=0).

streams property

The names of the registered streams.

aclose() async

Stop the listener and disconnect every viewer of every stream.

Source code in src/pdum/rfb/server.py 
571
572
573
574
575
576
577
578
579
580
581
async def aclose(self) -> None:
    """Stop the listener and disconnect every viewer of every stream."""
    if self._closed:
        return
    self._closed = True
    if self._listener_cm is not None:
        cm, self._listener_cm = self._listener_cm, None
        self._listener = None
        await cm.__aexit__(None, None, None)
    for host in self._streams.values():
        host.display._close_local()

add_stream(name, width, height, *, fps=30, bitrate=12000000, max_inflight=2, has_h264=None, has_nvenc=None, gpu=False, adaptive=False, still_after=None, stats_interval=None, authenticate=None, record_events=False, event_log=None, event_queue_size=4096, own_frames=False, resize_policy='publisher', max_render_dimension=None, encode_pipeline_depth=0)

Register a new named stream and return its :class:Display.

Streams are independent: each carries its own encoder config (one GPU, one image; per-stream bitrate; per-stream authenticate). Safe to call before or after :meth:start — clients reach it at ws://host/<name> either way. Raises if name is already taken. own_frames / resize_policy / max_render_dimension are forwarded to the stream's :class:Display.

Source code in src/pdum/rfb/server.py 
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
def add_stream(
    self,
    name: str,
    width: int,
    height: int,
    *,
    fps: int = 30,
    bitrate: int = 12_000_000,
    max_inflight: int = 2,
    has_h264: bool | None = None,
    has_nvenc: bool | None = None,
    gpu: bool = False,
    adaptive: bool = False,
    still_after: float | None = None,
    stats_interval: float | None = None,
    authenticate: Authenticator | None = None,
    record_events: bool = False,
    event_log: str | Path | None = None,
    event_queue_size: int = 4096,
    own_frames: bool = False,
    resize_policy: str = "publisher",
    max_render_dimension: int | None = None,
    encode_pipeline_depth: int = 0,
) -> Display:
    """Register a new named stream and return its :class:`Display`.

    Streams are independent: each carries its own encoder config (one GPU, one
    image; per-stream bitrate; per-stream ``authenticate``). Safe to call before
    or after :meth:`start` — clients reach it at ``ws://host/<name>`` either way.
    Raises if ``name`` is already taken. ``own_frames`` / ``resize_policy`` /
    ``max_render_dimension`` are forwarded to the stream's :class:`Display`.
    """
    if name in self._streams:
        raise ValueError(f"stream {name!r} already exists")
    display = Display(
        width,
        height,
        fps=fps,
        record_events=record_events,
        event_log=event_log,
        event_queue_size=event_queue_size,
        own_frames=own_frames,
        resize_policy=resize_policy,
        max_render_dimension=max_render_dimension,
    )
    host = _StreamHost(
        display,
        name,
        has_h264=has_h264,
        has_nvenc=has_nvenc,
        fps=fps,
        bitrate=bitrate,
        max_inflight=max_inflight,
        adaptive=adaptive,
        still_after=still_after,
        stats_interval=stats_interval,
        authenticate=authenticate,
        gpu=gpu,
        encode_pipeline_depth=encode_pipeline_depth,
    )
    self._streams[name] = host
    display._owner_server = self
    display._stream_name = name
    display._server = self._listener  # None until start(); back-filled there
    return display

process_request(connection, request)

Answer the HTTP side-channel routes; return None to proceed with WS.

Global: GET /health, GET /streams, GET /streams/<name>/metrics. For backward compatibility the single-stream routes (/metrics, /recorded-events, /recorded-events/reset) act on the "default" stream when one exists.

Source code in src/pdum/rfb/server.py 
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
def process_request(self, connection: Any, request: Any):
    """Answer the HTTP side-channel routes; return None to proceed with WS.

    Global: ``GET /health``, ``GET /streams``, ``GET /streams/<name>/metrics``.
    For backward compatibility the single-stream routes (``/metrics``,
    ``/recorded-events``, ``/recorded-events/reset``) act on the ``"default"``
    stream when one exists.
    """
    path = request.path.split("?", 1)[0]
    if path == "/health":
        return connection.respond(HTTPStatus.OK, "ok\n")
    if path == "/streams":
        return connection.respond(HTTPStatus.OK, json.dumps([h.info() for h in self._streams.values()]))
    parts = path.strip("/").split("/")
    if len(parts) == 3 and parts[0] == "streams" and parts[2] == "metrics":
        host = self._streams.get(parts[1])
        if host is None:
            return connection.respond(HTTPStatus.NOT_FOUND, "[]")
        return connection.respond(HTTPStatus.OK, json.dumps(host.metrics()))
    default = self._streams.get(DEFAULT_STREAM)
    if default is not None:
        if path == "/metrics":
            return connection.respond(HTTPStatus.OK, json.dumps(default.metrics()))
        if path == "/recorded-events":
            return connection.respond(HTTPStatus.OK, json.dumps(default.display.recorded))
        if path == "/recorded-events/reset":
            default.display.recorded.clear()
            return connection.respond(HTTPStatus.OK, "[]")
    return None

remove_stream(name)

Remove a named stream, disconnecting its viewers — the inverse of :meth:add_stream.

No-op if name is absent. The stream's :class:Display is closed locally (viewers dropped, waiters woken) but the shared listener keeps running for the other streams; a later connection to name closes with 4404. Used by the demo hub to reap idle per-client streams.

Source code in src/pdum/rfb/server.py 
531
532
533
534
535
536
537
538
539
540
541
def remove_stream(self, name: str) -> None:
    """Remove a named stream, disconnecting its viewers — the inverse of :meth:`add_stream`.

    No-op if ``name`` is absent. The stream's :class:`Display` is closed locally
    (viewers dropped, waiters woken) but the shared listener keeps running for the
    other streams; a later connection to ``name`` closes with ``4404``. Used by the
    demo hub to reap idle per-client streams.
    """
    host = self._streams.pop(name, None)
    if host is not None:
        host.display._close_local()

start() async

Start the shared listener in the background; returns self.

Source code in src/pdum/rfb/server.py 
557
558
559
560
561
562
563
564
565
566
567
568
569
async def start(self) -> Server:
    """Start the shared listener in the background; returns ``self``."""
    import websockets.asyncio.server

    kwargs: dict[str, Any] = dict(process_request=self.process_request, max_size=None)
    if self.origins is not None:
        kwargs["origins"] = self.origins
    cm = websockets.asyncio.server.serve(self._route, self.host, self._port, **kwargs)
    self._listener = await cm.__aenter__()
    self._listener_cm = cm
    for host in self._streams.values():
        host.display._server = self._listener
    return self

stream(name=DEFAULT_STREAM)

Return the :class:Display for stream name (KeyError if absent).

Source code in src/pdum/rfb/server.py 
527
528
529
def stream(self, name: str = DEFAULT_STREAM) -> Display:
    """Return the :class:`Display` for stream ``name`` (``KeyError`` if absent)."""
    return self._streams[name].display

serve(width, height, *, host='127.0.0.1', port=8765, fps=30, bitrate=12000000, max_inflight=2, has_h264=None, has_nvenc=None, gpu=False, adaptive=False, still_after=None, stats_interval=None, authenticate=None, origins=None, record_events=False, event_log=None, event_queue_size=4096, own_frames=False, resize_policy='publisher', max_render_dimension=None, encode_pipeline_depth=0) async

Start the RFB WebSocket server in the background and return a :class:Display.

You own your loop: display = await serve(w, h, port=...) then call display.publish(frame) whenever you like, and await display.aclose() to shut down.

Parameters:

Name Type Description Default
width int

Initial framebuffer size (a connecting client is configured to the display's current size; publish a different shape to resize).

 required
height int

Initial framebuffer size (a connecting client is configured to the display's current size; publish a different shape to resize).

 required
has_h264 bool | None

None auto-detects; False forces the CPU/image fallback. has_nvenc selects the GPU encoder when an NVENC-capable device is present.

 None
has_nvenc bool | None

None auto-detects; False forces the CPU/image fallback. has_nvenc selects the GPU encoder when an NVENC-capable device is present.

 None
gpu bool

Opt in to GPU encode: the publisher pushes CUDA frames (CuPy/DLPack NV12 or rgb) and each viewer's H.264 encoder reads them directly, no host copy. Prefers the PyAV-free NVENC SDK backend (habemus-papadum-nvenc) when available, else the zero-copy CUDA→NVENC backend (PyAV >= 18). Validated at startup; raises if neither is usable. For the PyAV-18 path, call :func:pdum.rfb.gpu.enable_cuda_context_sharing before any CuPy use.

 False
still_after float | None

Opt in to "still after interaction settles": when no new frame is published for still_after seconds (e.g. 0.15), each viewer is sent a high-quality still of the resting frame — a lossless PNG on the image path, a clean IDR on the video path — so the settled image is crisp even though the live stream is lossy. None (default) disables it. See docs/still_after_settle.md.

 None
adaptive bool

Enable adaptive quality (bitrate → fps → in-flight, with recovery): the encoder is rebuilt as the controller reacts to the client's decode-queue depth and RTT. Pairs well with stats_interval so the browser can show it.

 False
stats_interval float | None

Opt in to a periodic server→client stats control message (seconds, e.g. 1.0) carrying authoritative server metrics — RTT, fps, bitrate, encode time, and the adaptive targets — so the browser can surface them in its Stats. None (default) sends none.

 None
authenticate Authenticator | None

Optional async hook (see :mod:pdum.rfb.auth); rejected connections are closed with code 4401 before any frame is sent.

 None
origins list[str | None] | None

Allowed Origin values (CSWSH defense) passed to websockets.

 None
own_frames bool

Opt in to server-owned frames: publish() copies each frame into a recycled server buffer so you may reuse/mutate your own buffer immediately after it returns (no reallocation, no "released" callback). Default False keeps the zero-copy borrow (publish a fresh buffer each call). cpu/cuda only; metal raises. See :meth:Display.publish.

 False
resize_policy str

Opt in to match-client resize: resize_policy="match_client" makes a viewer's set_viewport authoritative — the render stream follows the viewport via display.target_size (default "publisher" keeps you in charge of size). max_render_dimension caps the followed size. See :attr:Display.target_size.

 'publisher'
max_render_dimension str

Opt in to match-client resize: resize_policy="match_client" makes a viewer's set_viewport authoritative — the render stream follows the viewport via display.target_size (default "publisher" keeps you in charge of size). max_render_dimension caps the followed size. See :attr:Display.target_size.

 'publisher'
encode_pipeline_depth int

Encoder pipeline depth. 0 (default) is synchronous 1-in-1-out — lowest latency, optimal for the interactive latest-frame-wins model. > 0 opts into the token-based pipelined encode path on backends that implement it (NVENC, for throughput at high fps / many streams). On VideoToolbox it is correct but not faster (low-latency RC is synchronous). See docs/pipelined_encode.md.

 0
Notes

This hosts a single "default" stream. Reach the hub behind it via display.server to host several streams from the one port (display.server.add_stream("camera_b", 640, 480)), or start with :func:serve_server for a hub with no default stream. See docs/multiple_streams.md.

Source code in src/pdum/rfb/server.py 
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
async def serve(
    width: int,
    height: int,
    *,
    host: str = "127.0.0.1",
    port: int = 8765,
    fps: int = 30,
    bitrate: int = 12_000_000,
    max_inflight: int = 2,
    has_h264: bool | None = None,
    has_nvenc: bool | None = None,
    gpu: bool = False,
    adaptive: bool = False,
    still_after: float | None = None,
    stats_interval: float | None = None,
    authenticate: Authenticator | None = None,
    origins: list[str | None] | None = None,
    record_events: bool = False,
    event_log: str | Path | None = None,
    event_queue_size: int = 4096,
    own_frames: bool = False,
    resize_policy: str = "publisher",
    max_render_dimension: int | None = None,
    encode_pipeline_depth: int = 0,
) -> Display:
    """Start the RFB WebSocket server in the background and return a :class:`Display`.

    You own your loop: ``display = await serve(w, h, port=...)`` then call
    ``display.publish(frame)`` whenever you like, and ``await display.aclose()`` to
    shut down.

    Parameters
    ----------
    width, height:
        Initial framebuffer size (a connecting client is configured to the
        display's current size; publish a different shape to resize).
    has_h264, has_nvenc:
        ``None`` auto-detects; ``False`` forces the CPU/image fallback. ``has_nvenc``
        selects the GPU encoder when an NVENC-capable device is present.
    gpu:
        Opt in to **GPU encode**: the publisher pushes CUDA frames (CuPy/DLPack NV12
        or rgb) and each viewer's H.264 encoder reads them directly, no host copy.
        Prefers the **PyAV-free NVENC SDK** backend (``habemus-papadum-nvenc``) when
        available, else the **zero-copy CUDA→NVENC** backend (PyAV >= 18). Validated
        at startup; raises if neither is usable. For the PyAV-18 path, call
        :func:`pdum.rfb.gpu.enable_cuda_context_sharing` before any CuPy use.
    still_after:
        Opt in to **"still after interaction settles"**: when no new frame is
        published for ``still_after`` seconds (e.g. ``0.15``), each viewer is sent a
        high-quality still of the resting frame — a **lossless PNG** on the image
        path, a clean **IDR** on the video path — so the settled image is crisp even
        though the live stream is lossy. ``None`` (default) disables it. See
        ``docs/still_after_settle.md``.
    adaptive:
        Enable adaptive quality (bitrate → fps → in-flight, with recovery): the
        encoder is rebuilt as the controller reacts to the client's decode-queue
        depth and RTT. Pairs well with ``stats_interval`` so the browser can show it.
    stats_interval:
        Opt in to a periodic server→client ``stats`` control message (seconds, e.g.
        ``1.0``) carrying authoritative server metrics — RTT, fps, bitrate, encode
        time, and the adaptive targets — so the browser can surface them in its
        ``Stats``. ``None`` (default) sends none.
    authenticate:
        Optional async hook (see :mod:`pdum.rfb.auth`); rejected connections are
        closed with code ``4401`` before any frame is sent.
    origins:
        Allowed ``Origin`` values (CSWSH defense) passed to ``websockets``.
    own_frames:
        Opt in to **server-owned frames**: ``publish()`` copies each frame into a
        recycled server buffer so you may reuse/mutate your own buffer immediately after
        it returns (no reallocation, no "released" callback). Default ``False`` keeps the
        zero-copy borrow (publish a fresh buffer each call). ``cpu``/``cuda`` only;
        ``metal`` raises. See :meth:`Display.publish`.
    resize_policy, max_render_dimension:
        Opt in to **match-client resize**: ``resize_policy="match_client"`` makes a viewer's
        ``set_viewport`` authoritative — the render stream follows the viewport via
        ``display.target_size`` (default ``"publisher"`` keeps you in charge of size).
        ``max_render_dimension`` caps the followed size. See :attr:`Display.target_size`.
    encode_pipeline_depth:
        Encoder pipeline depth. ``0`` (default) is synchronous 1-in-1-out — lowest
        latency, optimal for the interactive latest-frame-wins model. ``> 0`` opts into
        the token-based **pipelined** encode path on backends that implement it (NVENC,
        for throughput at high fps / many streams). On VideoToolbox it is correct but not
        faster (low-latency RC is synchronous). See ``docs/pipelined_encode.md``.

    Notes
    -----
    This hosts a single ``"default"`` stream. Reach the hub behind it via
    ``display.server`` to host **several** streams from the one port
    (``display.server.add_stream("camera_b", 640, 480)``), or start with
    :func:`serve_server` for a hub with no default stream. See
    ``docs/multiple_streams.md``.
    """
    server = Server(host=host, port=port, origins=origins)
    display = server.add_stream(
        DEFAULT_STREAM,
        width,
        height,
        fps=fps,
        bitrate=bitrate,
        max_inflight=max_inflight,
        has_h264=has_h264,
        has_nvenc=has_nvenc,
        gpu=gpu,
        adaptive=adaptive,
        still_after=still_after,
        stats_interval=stats_interval,
        authenticate=authenticate,
        record_events=record_events,
        event_log=event_log,
        event_queue_size=event_queue_size,
        own_frames=own_frames,
        resize_policy=resize_policy,
        max_render_dimension=max_render_dimension,
        encode_pipeline_depth=encode_pipeline_depth,
    )
    await server.start()
    # The one-liner contract: closing the returned Display tears down the whole hub.
    display._server_cm = server
    return display

serve_server(*, host='127.0.0.1', port=8765, origins=None, streams=None) async

Start a multi-stream hub and return the :class:Server.

Unlike :func:serve (one default stream, returns its Display), this returns the hub itself with no default stream. Add streams with server.add_stream(name, w, h, **config) — each returns its own Display to publish into — and clients attach by URL path (ws://host/<name>). A client with no path is rejected until a "default" stream exists.

Parameters:

Name Type Description Default
host str

As for :func:serve — they configure the one shared listener.

 '127.0.0.1'
port str

As for :func:serve — they configure the one shared listener.

 '127.0.0.1'
origins str

As for :func:serve — they configure the one shared listener.

 '127.0.0.1'
streams list[dict[str, Any]] | None

Optional list of add_stream keyword dicts to register before the listener starts (atomic setup), e.g. [{"name": "rgb", "width": 1280, "height": 720, "gpu": True}]. You can also add streams afterwards.

 None

Examples:

>>> server = await serve_server(port=8765)
>>> cam = server.add_stream("camera", 1280, 720)
>>> depth = server.add_stream("depth", 640, 480, has_h264=False)
>>> cam.publish(render_camera()); depth.publish(render_depth())
>>> await server.aclose()
Source code in src/pdum/rfb/server.py 
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
async def serve_server(
    *,
    host: str = "127.0.0.1",
    port: int = 8765,
    origins: list[str | None] | None = None,
    streams: list[dict[str, Any]] | None = None,
) -> Server:
    """Start a **multi-stream hub** and return the :class:`Server`.

    Unlike :func:`serve` (one default stream, returns its ``Display``), this returns
    the hub itself with no default stream. Add streams with
    ``server.add_stream(name, w, h, **config)`` — each returns its own ``Display`` to
    publish into — and clients attach by URL path (``ws://host/<name>``). A client
    with no path is rejected until a ``"default"`` stream exists.

    Parameters
    ----------
    host, port, origins:
        As for :func:`serve` — they configure the one shared listener.
    streams:
        Optional list of ``add_stream`` keyword dicts to register **before** the
        listener starts (atomic setup), e.g.
        ``[{"name": "rgb", "width": 1280, "height": 720, "gpu": True}]``. You can
        also add streams afterwards.

    Examples
    --------
    >>> server = await serve_server(port=8765)
    >>> cam = server.add_stream("camera", 1280, 720)
    >>> depth = server.add_stream("depth", 640, 480, has_h264=False)
    >>> cam.publish(render_camera()); depth.publish(render_depth())
    >>> await server.aclose()
    """
    server = Server(host=host, port=port, origins=origins)
    for spec in streams or []:
        server.add_stream(**spec)
    await server.start()
    return server

session

The WebSocket session loop and its backpressure / keyframe policy.

The policy matters more than the plumbing (guide sections 9 and 10):

  • latest-frame-wins backpressure: never keep more than max_inflight payloads unacknowledged; drop stale frames rather than letting latency grow;
  • the first payload to a new client is a keyframe, and a keyframe is forced again after any drop and on an explicit request_keyframe;
  • video encoders are fixed-resolution, so the encoder is rebuilt (and a keyframe forced) whenever the incoming frame size changes.

CPU-bound encoding runs in a worker thread via :func:asyncio.to_thread so the receive loop keeps draining ACKs, and the two loops run under a :class:asyncio.TaskGroup for clean structured shutdown.

RfbSession

Drive one client connection: encode + send frames, receive events.

Source code in src/pdum/rfb/session.py 
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
class RfbSession:
    """Drive one client connection: encode + send frames, receive events."""

    def __init__(
        self,
        source: FrameSource,
        encoder: EncoderBackend,
        ws: Any,
        *,
        encoder_factory: EncoderFactory | None = None,
        max_inflight: int = 2,
        bitrate: int = 12_000_000,
        fps: int = 30,
        adaptive: AdaptiveQualityController | None = None,
        still_after: float | None = None,
        stats_interval: float | None = None,
        inflight_timeout: float = 2.0,
        clock: Callable[[], float] | None = None,
    ) -> None:
        self.source = source
        self.encoder = encoder
        self.ws = ws
        self.encoder_factory = encoder_factory
        self.max_inflight = max_inflight
        self.bitrate = bitrate
        self.fps = fps
        self.adaptive = adaptive
        self.still_after = still_after
        self.stats_interval = stats_interval
        # Defense-in-depth backstop (docs/proposals/completed/client_decode_resilience.md):
        # if a sent seq sits unacked longer than this, the client is wedged (a stalled
        # decoder, an old build). Clear inflight + force a keyframe rather than dropping
        # every frame forever — this breaks the client↔server deadlock from the server side.
        self.inflight_timeout = inflight_timeout
        self.inflight_timeouts = 0  # count of backstop trips (for tests / metrics)
        self.decoder_resets = 0  # count of client decoder_reset controls honored
        self._last_stats_t = -1e9
        self._clock = clock or time.monotonic

        self.force_keyframe = True
        # Adaptive resolution lever (serve(adaptive=True)): the controller's current
        # recommended render scale. The session cannot resize the shared framebuffer
        # itself, so a change is surfaced to the publisher via the source (a DownscaleHint
        # through Display.poll_events()); starts at full resolution.
        self.render_scale = 1.0
        self.inflight: set[int] = set()
        self.dropped = 0
        self._last_drop_log_t = -1e9  # rate-limit the "dropping frames" warning to ~1/s
        self.closed = False
        self._enc_size: tuple[int, int] | None = None
        self._send_times: dict[int, float] = {}
        self.metrics = SessionMetrics(started_at=self._clock(), target_bitrate=bitrate, target_fps=fps)

        # "Still after interaction settles": when the scene goes quiet (no new
        # published frame for ``still_after`` seconds), re-send the resting frame
        # at higher quality — a lossless PNG on the image path, a clean IDR on the
        # video path. Opt-in, and only when both the source can produce a still
        # and the encoder knows how to encode one (otherwise a silent no-op).
        self._still_pending = False
        self._stills_enabled = (
            still_after is not None and hasattr(encoder, "encode_still") and hasattr(source, "still_frame")
        )
        # Server-owned buffers the resting frame is snapshotted into before the off-thread
        # still encode, so a caller that reuses/mutates its published buffer while idle can't
        # corrupt the still. Allocated once and reused; reallocated only on a size/dtype change.
        # See _snapshot_still and docs/still_after_settle.md.
        self._still_buf: np.ndarray | None = None
        self._still_buf_cuda: Any = None

    def metrics_snapshot(self) -> dict:
        """Return a JSON-serializable snapshot of this session's metrics."""
        self.metrics.inflight = len(self.inflight)
        self.metrics.target_bitrate = self.bitrate
        self.metrics.target_fps = self.fps
        return self.metrics.snapshot(now=self._clock())

    # --- receive side -------------------------------------------------------

    def _reset_inflight(self) -> None:
        """Clear the unacked-payload set so the encode loop can send again, and force the next
        frame to a self-contained keyframe. Shared by the client ``decoder_reset`` control and
        the server-side inflight-timeout backstop. It does **not** ACK the stalled seqs (that
        would corrupt the displayed-FIFO seq attribution + RTT); it just stops waiting."""
        self.inflight.clear()
        self._send_times.clear()
        self.metrics.inflight = 0
        self.force_keyframe = True

    async def _handle_control(self, data: dict) -> None:
        """Process one decoded JSON control message (one step of ``recv_loop``)."""
        kind = data.get("type")
        if kind == "ack":
            seq = data.get("seq")
            now = self._clock()
            sent_at = self._send_times.pop(seq, None)
            rtt_ms = (now - sent_at) * 1000 if sent_at is not None else None
            self.inflight.discard(seq)
            self.metrics.inflight = len(self.inflight)
            self.metrics.record_ack(
                rtt_ms=rtt_ms,
                decode_queue_size=int(data.get("decode_queue_size", 0)),
                now=now,
            )
            await self._maybe_send_stats(now)
        elif kind == "request_keyframe":
            self.force_keyframe = True
        elif kind == "decoder_reset":
            # The client's decoder stalled and it rebuilt — stop waiting for frames it will
            # never display so the next encode can send it a fresh keyframe.
            self.decoder_resets += 1
            log.info("client rebuilt its decoder (stall recovery) — clearing inflight + forcing keyframe")
            self._reset_inflight()
        elif kind == "event":
            await self.source.handle_event(data["event"])
        elif kind == "set_viewport":
            # Renderview-shaped resize: logical width/height, physical pwidth/pheight,
            # ratio. Older clients sent only width/height (physical) + pixel_ratio.
            await self.source.handle_event(
                {
                    "type": "resize",
                    "width": data["width"],
                    "height": data["height"],
                    "pwidth": data.get("pwidth", data["width"]),
                    "pheight": data.get("pheight", data["height"]),
                    "ratio": data.get("ratio", data.get("pixel_ratio", 1)),
                }
            )

    async def recv_loop(self) -> None:
        try:
            async for msg in self.ws:
                if isinstance(msg, (bytes, bytearray)):
                    continue
                await self._handle_control(parse_control(msg))
        except _ConnectionClosed:
            pass
        finally:
            # When the client disconnects, stop the encode loop too.
            self.closed = True

    # --- send side ----------------------------------------------------------

    async def send_payload(self, payload: EncodedPayload) -> None:
        await self.ws.send(pack_binary_message(header_for(payload), payload.payload))
        now = self._clock()
        self.inflight.add(payload.seq)
        self._send_times[payload.seq] = now
        self.metrics.inflight = len(self.inflight)
        self.metrics.record_sent(payload_bytes=len(payload.payload), keyframe=payload.keyframe, now=now)

    #: Pending live reconfigure (backend/transport/params). Set by
    #: :meth:`request_reconfigure` and applied at the top of an encode step so it never
    #: races the off-thread ``encoder.encode``. ``None`` when idle.
    _reconfig: dict | None = None

    def request_reconfigure(
        self,
        *,
        factory: EncoderFactory | None = None,
        selection: Any = None,
        bitrate: int | None = None,
        fps: int | None = None,
    ) -> None:
        """Queue a live encoder swap and/or quality change.

        Applied before the next encode (never mid-encode). ``factory`` swaps the encoder
        backend/transport; ``selection`` (a :class:`~pdum.rfb.protocol.BackendSelection`)
        triggers a fresh ``config`` to the client; ``bitrate``/``fps`` retune quality.
        Must be called on the event-loop thread (it only stores state).
        """
        self._reconfig = {"factory": factory, "selection": selection, "bitrate": bitrate, "fps": fps}

    async def _apply_reconfigure(self, width: int, height: int) -> None:
        """Apply a pending :meth:`request_reconfigure` between encode steps."""
        req, self._reconfig = self._reconfig, None
        if req is None:
            return
        if req["factory"] is not None:
            self.encoder_factory = req["factory"]
        if req["bitrate"] is not None:
            self.bitrate = req["bitrate"]
        if req["fps"] is not None:
            self.fps = req["fps"]
        if self.encoder_factory is not None:
            self.encoder.close()
            self.encoder = self.encoder_factory(width, height, self.bitrate, self.fps)
            self._enc_size = (width, height)
            self.force_keyframe = True  # next frame is a self-contained keyframe (KeyframeGate)
        sel = req["selection"]
        if sel is not None:
            transport = "webcodecs" if sel.transport == "h264" else "image"
            await self.ws.send(config_message(transport=transport, width=width, height=height, codec=sel.codec))

    def _ensure_encoder_for(self, width: int, height: int) -> None:
        """Rebuild the (fixed-resolution) encoder if the frame size changed."""
        size = (width, height)
        if self._enc_size is None:
            self._enc_size = size
            return
        if size != self._enc_size and self.encoder_factory is not None:
            self.encoder.close()
            self.encoder = self.encoder_factory(width, height, self.bitrate, self.fps)
            self.force_keyframe = True
            self._enc_size = size

    async def _encode_step(self) -> str:
        """Run one encode iteration.

        Returns ``"sent"``, ``"dropped"``, ``"still"`` or ``"stopped"``.
        """
        try:
            frame = await self._next_frame_or_idle()
        except StopAsyncIteration:
            return "stopped"

        if frame is None:
            # The scene settled: upgrade the resting frame to a high-quality still.
            await self._send_still()
            return "still"

        # A fresh frame supersedes any still we were about to send; arm the next.
        self._still_pending = self._stills_enabled

        # Latest-frame-wins: if the client is behind, drop this frame before
        # spending encode time and force the next sent one to be a keyframe.
        if len(self.inflight) >= self.max_inflight:
            now = self._clock()
            oldest = min(self._send_times.values(), default=now)
            if self._send_times and now - oldest > self.inflight_timeout:
                # Backstop: nothing acked for inflight_timeout — the client is wedged (a
                # stalled decoder / old build). Clear inflight + force a keyframe and fall
                # through to send *this* frame, instead of dropping forever (silent deadlock).
                self.inflight_timeouts += 1
                log.warning(
                    "client unresponsive for %.1fs (wedged decoder?) — clearing inflight + forcing keyframe",
                    now - oldest,
                )
                self._reset_inflight()
            else:
                self.dropped += 1
                self.force_keyframe = True
                self.metrics.record_dropped(now=now)
                # Rate-limited so a sustained backlog logs ~once/second, not per dropped frame.
                if now - self._last_drop_log_t >= 1.0:
                    log.warning(
                        "client behind — dropping frames (%d dropped so far; next frame forced to keyframe)",
                        self.dropped,
                    )
                    self._last_drop_log_t = now
                return "dropped"

        if self._reconfig is not None:
            await self._apply_reconfigure(frame.width, frame.height)
        self._ensure_encoder_for(frame.width, frame.height)
        force = self.force_keyframe
        t0 = self._clock()
        payloads = await asyncio.to_thread(self.encoder.encode, frame, force_keyframe=force)
        self.metrics.record_encode((self._clock() - t0) * 1000, now=self._clock())
        self.force_keyframe = False

        for payload in payloads:
            self._stamp_render_descriptors(payload, frame)
            await self.send_payload(payload)
        await self._maybe_adapt()
        return "sent"

    async def _next_frame_or_idle(self) -> Any:
        """Park for the next frame; surface a settle window as ``None``.

        When stills are enabled and one is pending, the wait is bounded by
        ``still_after`` so that ``still_after`` seconds without a new published
        frame returns ``None`` ("the scene settled — send a still"). Otherwise it
        blocks until the next frame, exactly like the plain pull. The pending flag
        is cleared once a still fires, so the bounded wait happens at most once per
        settle and the loop reverts to a blocking park (no busy-loop on idle).
        """
        if not (self._stills_enabled and self._still_pending):
            return await self.source.next_frame()
        try:
            return await asyncio.wait_for(self.source.next_frame(), self.still_after)
        except TimeoutError:
            return None

    async def _send_still(self) -> None:
        """Encode and send a lossless / high-quality still of the settled frame.

        The still re-sends the *current latest* frame (the one the client is
        resting on) with a fresh per-client ``seq`` and as a self-contained
        keyframe, so a client that dropped deltas during a flurry also jumps
        straight to the latest. A one-shot nicety, so it is skipped (rather than
        queued) when the client is still catching up.
        """
        self._still_pending = False
        still = self.source.still_frame()  # type: ignore[attr-defined]
        if still is None or len(self.inflight) >= self.max_inflight:
            return
        self._ensure_encoder_for(still.width, still.height)
        # Snapshot the resting frame into a server-owned buffer on THIS (loop) thread, so the
        # off-thread encode below reads a stable copy even if the caller reuses/mutates its
        # published buffer while the scene is settled. No await between here and to_thread, so
        # the copy is atomic w.r.t. the generator (which publishes on the same thread).
        still = self._snapshot_still(still)
        t0 = self._clock()
        payloads = await asyncio.to_thread(self.encoder.encode_still, still)  # type: ignore[attr-defined]
        self.metrics.record_encode((self._clock() - t0) * 1000, now=self._clock())
        for payload in payloads:
            self._stamp_render_descriptors(payload, still)
            await self.send_payload(payload)

    @staticmethod
    def _stamp_render_descriptors(payload: EncodedPayload, frame: RawFrame) -> None:
        """Carry the source frame's render-side descriptors (``pixel_ratio`` / ``color``)
        onto the outgoing payload, so ``header_for`` can emit them without any encoder
        needing to know they exist. The session is the single frame→payload seam."""
        payload.pixel_ratio = frame.pixel_ratio
        payload.color = frame.color

    def _snapshot_still(self, frame: RawFrame) -> RawFrame:
        """Copy the resting ``frame`` into a server-owned, reused buffer for the still encode.

        Runs on the loop thread, so it is atomic w.r.t. the generator (which publishes on the
        same thread); the off-thread ``encode_still`` then reads this stable copy rather than
        the caller's buffer. The buffer is allocated once and reused; a size/dtype change
        reallocates it (mirrors the fixed-resolution encoder rebuild in ``_ensure_encoder_for``).

        Metal frames are returned unchanged: MLX arrays are functionally immutable (a render
        yields a *new* array) and ``publish()`` already materializes them on the loop thread, so
        there is no torn read to sever. See docs/still_after_settle.md.
        """
        data = frame.data
        if frame.memory == "cpu":
            arr = np.asarray(data)
            if self._still_buf is None or self._still_buf.shape != arr.shape or self._still_buf.dtype != arr.dtype:
                self._still_buf = np.empty(arr.shape, arr.dtype)  # C-contiguous, not empty_like
            np.copyto(self._still_buf, arr)
            return dataclasses.replace(frame, data=self._still_buf)
        if frame.memory == "cuda":
            import cupy as cp  # lazy: only when a cuda frame settles, so ``import pdum.rfb`` stays dep-free

            src = cp.asarray(data)
            buf = self._still_buf_cuda
            if buf is None or buf.shape != src.shape or buf.dtype != src.dtype:
                buf = self._still_buf_cuda = cp.empty(src.shape, src.dtype)
            cp.copyto(buf, src)
            return dataclasses.replace(frame, data=buf)
        return frame  # metal (MLX immutable) / other: no snapshot needed

    async def _maybe_send_stats(self, now: float) -> None:
        """Push a server-truth ``stats`` control message to the client, throttled.

        Opt-in via ``stats_interval``: lets the browser surface authoritative
        server-side metrics (RTT, fps, bitrate, encode time, the adaptive targets)
        in its ``Stats`` — the client only sees its own decode side otherwise.
        """
        if self.stats_interval is None or now - self._last_stats_t < self.stats_interval:
            return
        self._last_stats_t = now
        snap = self.metrics_snapshot()
        await self.ws.send(
            json.dumps(
                {
                    "type": "stats",
                    "rtt_ms": snap["rtt_ms"],
                    "fps_sent": snap["fps_sent"],
                    "fps_acked": snap["fps_acked"],
                    "bitrate_bps": snap["bitrate_bps"],
                    "encode_ms": snap["encode_ms"],
                    "decode_queue_size": snap["decode_queue_size"],
                    "dropped": snap["frames_dropped"],
                    "target_bitrate": snap["target_bitrate"],
                    "target_fps": snap["target_fps"],
                }
            )
        )

    async def _maybe_adapt(self) -> None:
        """Apply an adaptive-quality decision, if the controller requests one."""
        if self.adaptive is None:
            return
        target = self.adaptive.update(self.metrics_snapshot(), now=self._clock())
        if target is None:
            return
        self.max_inflight = target.max_inflight
        rebuild = target.bitrate != self.bitrate or target.fps != self.fps
        self.bitrate = target.bitrate
        self.fps = target.fps
        if rebuild and self.encoder_factory is not None and self._enc_size is not None:
            w, h = self._enc_size
            self.encoder.close()
            self.encoder = self.encoder_factory(w, h, self.bitrate, self.fps)
            self.force_keyframe = True
        # Resolution lever: the session can't resize the shared framebuffer, so hand the
        # recommendation to the publisher via the source (Display fans out a DownscaleHint).
        if target.scale != self.render_scale:
            self.render_scale = target.scale
            suggest = getattr(self.source, "suggest_render_scale", None)
            if suggest is not None:
                suggest(target.scale)
        await self.ws.send(json.dumps({"type": "set_quality", "bitrate": self.bitrate, "fps": self.fps}))

    async def encode_loop(self) -> None:
        try:
            while not self.closed:
                result = await self._encode_step()
                if result == "stopped":
                    break
                if result == "dropped":
                    await asyncio.sleep(0)
        except _ConnectionClosed:
            self.closed = True

    async def run(self) -> None:
        """Run the receive and encode loops until the connection closes."""
        try:
            async with asyncio.TaskGroup() as tg:
                tg.create_task(self.recv_loop())
                tg.create_task(self.encode_loop())
        finally:
            self.closed = True
            self.encoder.close()

metrics_snapshot()

Return a JSON-serializable snapshot of this session's metrics.

Source code in src/pdum/rfb/session.py 
113
114
115
116
117
118
def metrics_snapshot(self) -> dict:
    """Return a JSON-serializable snapshot of this session's metrics."""
    self.metrics.inflight = len(self.inflight)
    self.metrics.target_bitrate = self.bitrate
    self.metrics.target_fps = self.fps
    return self.metrics.snapshot(now=self._clock())

request_reconfigure(*, factory=None, selection=None, bitrate=None, fps=None)

Queue a live encoder swap and/or quality change.

Applied before the next encode (never mid-encode). factory swaps the encoder backend/transport; selection (a :class:~pdum.rfb.protocol.BackendSelection) triggers a fresh config to the client; bitrate/fps retune quality. Must be called on the event-loop thread (it only stores state).

Source code in src/pdum/rfb/session.py 
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
def request_reconfigure(
    self,
    *,
    factory: EncoderFactory | None = None,
    selection: Any = None,
    bitrate: int | None = None,
    fps: int | None = None,
) -> None:
    """Queue a live encoder swap and/or quality change.

    Applied before the next encode (never mid-encode). ``factory`` swaps the encoder
    backend/transport; ``selection`` (a :class:`~pdum.rfb.protocol.BackendSelection`)
    triggers a fresh ``config`` to the client; ``bitrate``/``fps`` retune quality.
    Must be called on the event-loop thread (it only stores state).
    """
    self._reconfig = {"factory": factory, "selection": selection, "bitrate": bitrate, "fps": fps}

run() async

Run the receive and encode loops until the connection closes.

Source code in src/pdum/rfb/session.py 
455
456
457
458
459
460
461
462
463
async def run(self) -> None:
    """Run the receive and encode loops until the connection closes."""
    try:
        async with asyncio.TaskGroup() as tg:
            tg.create_task(self.recv_loop())
            tg.create_task(self.encode_loop())
    finally:
        self.closed = True
        self.encoder.close()

sources

Frame sources: base bookkeeping plus a render-callback adapter.

A :class:FrameSource produces raw frames and consumes user-input events. :class:BaseFrameSource owns the boring-but-easy-to-get-wrong bookkeeping (sequence numbers, microsecond timestamps, fps pacing, event recording and viewport tracking) so concrete sources only implement :meth:render.

The deterministic, GUI-free :class:~pdum.rfb.testing.SyntheticFrameSource used by the tests and the demo server lives in :mod:pdum.rfb.testing.

BaseFrameSource

Bases: ABC

Common bookkeeping for frame sources.

Parameters:

Name Type Description Default
width int

Initial framebuffer size in pixels (forced even for codec friendliness).

 640
height int

Initial framebuffer size in pixels (forced even for codec friendliness).

 640
fps int

Target frame rate used for pacing when pace is true.

 30
pixel_format PixelFormat

"rgb24" or "rgba8"; :meth:render must return matching arrays.

 'rgb24'
max_frames int | None

If set, :meth:next_frame raises StopAsyncIteration after this many.

 None
pace bool

When true, :meth:next_frame sleeps to approximate fps. Tests set this false to run as fast as possible.

 True
clock Callable[[], float] | None

Monotonic clock returning seconds; injectable for deterministic tests.

 None
Source code in src/pdum/rfb/sources.py 
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
class BaseFrameSource(ABC):
    """Common bookkeeping for frame sources.

    Parameters
    ----------
    width, height:
        Initial framebuffer size in pixels (forced even for codec friendliness).
    fps:
        Target frame rate used for pacing when ``pace`` is true.
    pixel_format:
        ``"rgb24"`` or ``"rgba8"``; :meth:`render` must return matching arrays.
    max_frames:
        If set, :meth:`next_frame` raises ``StopAsyncIteration`` after this many.
    pace:
        When true, :meth:`next_frame` sleeps to approximate ``fps``. Tests set
        this false to run as fast as possible.
    clock:
        Monotonic clock returning seconds; injectable for deterministic tests.
    """

    def __init__(
        self,
        *,
        width: int = 640,
        height: int = 480,
        fps: int = 30,
        pixel_format: PixelFormat = "rgb24",
        max_frames: int | None = None,
        pace: bool = True,
        clock: Callable[[], float] | None = None,
    ) -> None:
        self.width = _make_even(width)
        self.height = _make_even(height)
        self.fps = fps
        self.pixel_format = pixel_format
        self.max_frames = max_frames
        self.pace = pace
        self._clock = clock or time.monotonic

        self.seq = 0
        self.frames_produced = 0
        self.events: list[EventDict] = []
        self._start = self._clock()
        self._next_due = self._start

    @property
    def current_size(self) -> tuple[int, int]:
        return (self.width, self.height)

    @abstractmethod
    def render(self, seq: int, t_us: int) -> np.ndarray:
        """Return the pixel array for frame ``seq`` at timestamp ``t_us``.

        Must be deterministic in ``seq`` and match ``self.pixel_format``.
        """

    def _produce_frame(self) -> RawFrame:
        """Render the current frame with a real monotonic timestamp."""
        t_us = int((self._clock() - self._start) * 1_000_000)
        arr = self.render(self.seq, t_us)
        frame = RawFrame(
            seq=self.seq,
            width=self.width,
            height=self.height,
            timestamp_us=t_us,
            pixel_format=self.pixel_format,
            memory="cpu",
            data=arr,
        )
        self.seq += 1
        self.frames_produced += 1
        return frame

    async def next_frame(self) -> RawFrame:
        if self.max_frames is not None and self.frames_produced >= self.max_frames:
            raise StopAsyncIteration

        if self.pace:
            self._next_due += 1.0 / self.fps
            delay = self._next_due - self._clock()
            if delay > 0:
                await asyncio.sleep(delay)

        return self._produce_frame()

    async def handle_event(self, event: EventDict) -> None:
        recorded = dict(event)
        recorded["_received_us"] = int((self._clock() - self._start) * 1_000_000)
        self.events.append(recorded)
        if event.get("type") == "resize":
            # Render at the physical (backing-store) size; renderview carries it as
            # pwidth/pheight, older clients only sent width/height.
            self.width = _make_even(int(event.get("pwidth", event["width"])))
            self.height = _make_even(int(event.get("pheight", event["height"])))

    def snapshot_events(self) -> list[EventDict]:
        """Return a copy of all events received so far, in order."""
        return list(self.events)

render(seq, t_us) abstractmethod

Return the pixel array for frame seq at timestamp t_us.

Must be deterministic in seq and match self.pixel_format.

Source code in src/pdum/rfb/sources.py 
73
74
75
76
77
78
@abstractmethod
def render(self, seq: int, t_us: int) -> np.ndarray:
    """Return the pixel array for frame ``seq`` at timestamp ``t_us``.

    Must be deterministic in ``seq`` and match ``self.pixel_format``.
    """

snapshot_events()

Return a copy of all events received so far, in order.

Source code in src/pdum/rfb/sources.py 
119
120
121
def snapshot_events(self) -> list[EventDict]:
    """Return a copy of all events received so far, in order."""
    return list(self.events)

OnDemandFrameSource

Bases: BaseFrameSource

A sparse, event-driven source that renders only when marked dirty.

For scientific visualization the framebuffer often changes only when the user interacts or a parameter updates (guide addendum, section 1). Instead of fabricating duplicate frames at a fixed rate, :meth:next_frame parks until :meth:mark_dirty is called (or, by default, until an input event arrives), then emits a single frame with a real timestamp. The session's latest-frame- wins policy and the encoder's keyframe handling are unchanged.

Parameters:

Name Type Description Default
render Callable[[int, int], ndarray]

render(seq, t_us) -> ndarray producing the current frame.

 required
render_on_event bool

When true (default), any received input event marks the source dirty so interaction re-renders automatically.

 True
Source code in src/pdum/rfb/sources.py 
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
class OnDemandFrameSource(BaseFrameSource):
    """A sparse, event-driven source that renders only when marked dirty.

    For scientific visualization the framebuffer often changes only when the user
    interacts or a parameter updates (guide addendum, section 1). Instead of
    fabricating duplicate frames at a fixed rate, :meth:`next_frame` parks until
    :meth:`mark_dirty` is called (or, by default, until an input event arrives),
    then emits a single frame with a real timestamp. The session's latest-frame-
    wins policy and the encoder's keyframe handling are unchanged.

    Parameters
    ----------
    render:
        ``render(seq, t_us) -> ndarray`` producing the current frame.
    render_on_event:
        When true (default), any received input event marks the source dirty so
        interaction re-renders automatically.
    """

    def __init__(
        self,
        render: Callable[[int, int], np.ndarray],
        *,
        render_on_event: bool = True,
        **kwargs,
    ) -> None:
        kwargs.setdefault("pace", False)
        super().__init__(**kwargs)
        self._render_fn = render
        self._render_on_event = render_on_event
        self._dirty = asyncio.Event()
        self._dirty.set()  # emit one frame on connect

    def mark_dirty(self) -> None:
        """Request that the next :meth:`next_frame` produces a fresh frame."""
        self._dirty.set()

    def render(self, seq: int, t_us: int) -> np.ndarray:
        return self._render_fn(seq, t_us)

    async def next_frame(self) -> RawFrame:
        if self.max_frames is not None and self.frames_produced >= self.max_frames:
            raise StopAsyncIteration
        await self._dirty.wait()
        self._dirty.clear()
        return self._produce_frame()

    async def handle_event(self, event: EventDict) -> None:
        await super().handle_event(event)
        if self._render_on_event:
            self.mark_dirty()

mark_dirty()

Request that the next :meth:next_frame produces a fresh frame.

Source code in src/pdum/rfb/sources.py 
168
169
170
def mark_dirty(self) -> None:
    """Request that the next :meth:`next_frame` produces a fresh frame."""
    self._dirty.set()

RenderCallbackSource

Bases: BaseFrameSource

Adapt a plain render(seq, t_us) -> ndarray callable into a source.

Source code in src/pdum/rfb/sources.py 
124
125
126
127
128
129
130
131
132
class RenderCallbackSource(BaseFrameSource):
    """Adapt a plain ``render(seq, t_us) -> ndarray`` callable into a source."""

    def __init__(self, render: Callable[[int, int], np.ndarray], **kwargs) -> None:
        super().__init__(**kwargs)
        self._render = render

    def render(self, seq: int, t_us: int) -> np.ndarray:
        return self._render(seq, t_us)

testing

Test and demo helpers (omitted from coverage on purpose).

Contains the headless-rendering :class:SyntheticFrameSource, in-memory fakes for driving the session without real sockets/encoders, Annex B / NAL helpers for validating H.264 output, and a fixture generator that keeps the JavaScript wire protocol byte-compatible with the Python one.

The :func:render_test_pattern formula is the shared contract re-implemented in widgets/tests so the browser e2e can verify decoded pixels against a locally computed expectation.

FakeEncoder

A deterministic in-memory encoder for session tests (no PyAV needed).

Source code in src/pdum/rfb/testing.py 
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
class FakeEncoder:
    """A deterministic in-memory encoder for session tests (no PyAV needed)."""

    def __init__(self, *, kind: str = "video", keyframe_interval: int = 30, **_: object) -> None:
        self.kind = kind
        self.keyframe_interval = keyframe_interval
        self.calls = 0

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        is_key = force_keyframe or (self.calls % self.keyframe_interval == 0)
        self.calls += 1
        payload = f"frame:{frame.seq}:{'key' if is_key else 'delta'}".encode()
        return [
            EncodedPayload(
                seq=frame.seq,
                kind=self.kind,  # type: ignore[arg-type]
                timestamp_us=frame.timestamp_us,
                width=frame.width,
                height=frame.height,
                payload=payload,
                codec="avc1.42E01F" if self.kind == "video" else None,
                keyframe=is_key,
                metadata={"bitstream": "annexb"} if self.kind == "video" else None,
            )
        ]

    def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
        """A distinguishable 'still' payload for "still after settle" tests."""
        self.calls += 1
        return [
            EncodedPayload(
                seq=frame.seq,
                kind=self.kind,  # type: ignore[arg-type]
                timestamp_us=frame.timestamp_us,
                width=frame.width,
                height=frame.height,
                payload=f"still:{frame.seq}".encode(),
                codec="avc1.42E01F" if self.kind == "video" else None,
                keyframe=True,
                metadata={"bitstream": "annexb"} if self.kind == "video" else None,
            )
        ]

    def flush(self) -> list[EncodedPayload]:
        return []

    def close(self) -> None:
        pass

encode_still(frame)

A distinguishable 'still' payload for "still after settle" tests.

Source code in src/pdum/rfb/testing.py 
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
def encode_still(self, frame: RawFrame) -> list[EncodedPayload]:
    """A distinguishable 'still' payload for "still after settle" tests."""
    self.calls += 1
    return [
        EncodedPayload(
            seq=frame.seq,
            kind=self.kind,  # type: ignore[arg-type]
            timestamp_us=frame.timestamp_us,
            width=frame.width,
            height=frame.height,
            payload=f"still:{frame.seq}".encode(),
            codec="avc1.42E01F" if self.kind == "video" else None,
            keyframe=True,
            metadata={"bitstream": "annexb"} if self.kind == "video" else None,
        )
    ]

FakeWebSocket

In-memory duplex stand-in for a websockets connection.

sent collects every outbound message. Inbound messages are queued via :meth:inject and yielded by async iteration until :meth:close.

Source code in src/pdum/rfb/testing.py 
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
class FakeWebSocket:
    """In-memory duplex stand-in for a websockets connection.

    ``sent`` collects every outbound message. Inbound messages are queued via
    :meth:`inject` and yielded by async iteration until :meth:`close`.
    """

    def __init__(self) -> None:
        self.sent: list[bytes | str] = []
        self._inbound: asyncio.Queue[bytes | str] = asyncio.Queue()
        self._closed = False
        self.on_send: Callable[[bytes | str], None] | None = None

    async def send(self, data: bytes | str) -> None:
        self.sent.append(data)
        if self.on_send is not None:
            self.on_send(data)

    def inject(self, message: bytes | str | dict) -> None:
        if isinstance(message, dict):
            message = json.dumps(message)
        self._inbound.put_nowait(message)

    def close(self) -> None:
        self._closed = True
        # Wake a pending __anext__.
        self._inbound.put_nowait(_SENTINEL)

    def __aiter__(self) -> "FakeWebSocket":
        return self

    async def __anext__(self) -> bytes | str:
        if self._closed and self._inbound.empty():
            raise StopAsyncIteration
        item = await self._inbound.get()
        if item is _SENTINEL:
            raise StopAsyncIteration
        return item

SyntheticFrameSource

Bases: BaseFrameSource

Deterministic, GUI-free frame source for tests and the demo server.

Source code in src/pdum/rfb/testing.py 
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
class SyntheticFrameSource(BaseFrameSource):
    """Deterministic, GUI-free frame source for tests and the demo server."""

    def __init__(self, *, pattern: Pattern = "test_card", **kwargs) -> None:
        super().__init__(**kwargs)
        if pattern not in _PATTERNS:
            raise ValueError(f"unknown pattern {pattern!r}; choose from {sorted(_PATTERNS)}")
        self.pattern = pattern
        self._renderer = _PATTERNS[pattern]
        if self.pixel_format not in ("rgb24", "rgba8"):
            raise ValueError("SyntheticFrameSource supports rgb24 / rgba8 only")

    def render(self, seq: int, t_us: int) -> np.ndarray:
        rgb = self._renderer(seq, self.width, self.height)
        if self.pixel_format == "rgba8":
            rgba = np.empty((self.height, self.width, 4), dtype=np.uint8)
            rgba[..., :3] = rgb
            rgba[..., 3] = 255
            return np.ascontiguousarray(rgba)
        return np.ascontiguousarray(rgb)

decode_annexb(data)

Decode an H.264 Annex B byte stream back to frames using PyAV.

Returns a list of av.VideoFrame. Used by the headless encoder tests to prove the produced bitstream is valid and decodable without a browser.

Source code in src/pdum/rfb/testing.py 
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
def decode_annexb(data: bytes) -> list:
    """Decode an H.264 Annex B byte stream back to frames using PyAV.

    Returns a list of ``av.VideoFrame``. Used by the headless encoder tests to
    prove the produced bitstream is valid and decodable without a browser.
    """
    import av  # lazy: only needed when validating H.264 output

    frames = []
    codec = av.CodecContext.create("h264", "r")
    packets = codec.parse(data)
    for packet in packets:
        frames.extend(codec.decode(packet))
    frames.extend(codec.decode(None))  # flush
    return frames

expected_quadrant_color(seq, quadrant)

Return the expected RGB color of quadrant (0..3) at frame seq.

Source code in src/pdum/rfb/testing.py 
53
54
55
56
def expected_quadrant_color(seq: int, quadrant: int) -> tuple[int, int, int]:
    """Return the expected RGB color of ``quadrant`` (0..3) at frame ``seq``."""
    r, g, b = _QUADRANT_COLORS[(quadrant + seq) % 4]
    return (int(r), int(g), int(b))

gen_fixtures(out_dir)

Generate protocol parity fixtures for the JavaScript test suite.

Writes <name>.bin (the packed binary message) and <name>.json (the expected header + payload hex) for a few canonical messages.

Source code in src/pdum/rfb/testing.py 
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
def gen_fixtures(out_dir: str | Path) -> list[Path]:
    """Generate protocol parity fixtures for the JavaScript test suite.

    Writes ``<name>.bin`` (the packed binary message) and ``<name>.json``
    (the expected header + payload hex) for a few canonical messages.
    """
    out = Path(out_dir)
    out.mkdir(parents=True, exist_ok=True)

    cases: list[tuple[str, dict, bytes]] = [
        (
            "image_jpeg",
            {
                "type": "image_frame",
                "seq": 42,
                "timestamp_us": 700000,
                "width": 1280,
                "height": 720,
                "mime": "image/jpeg",
            },
            bytes([0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10]),
        ),
        (
            "video_annexb",
            {
                "type": "video_chunk",
                "seq": 7,
                "timestamp_us": 16666,
                "width": 640,
                "height": 480,
                "codec": "avc1.42E01F",
                "bitstream": "annexb",
                "keyframe": True,
            },
            bytes([0x00, 0x00, 0x00, 0x01, 0x67, 0x42]),
        ),
        (
            "unicode_header",
            {"type": "image_frame", "seq": 1, "note": "café-🎞", "width": 2, "height": 2},
            bytes([1, 2, 3]),
        ),
    ]

    written: list[Path] = []
    for name, header, payload in cases:
        packed = pack_binary_message(header, payload)
        bin_path = out / f"{name}.bin"
        json_path = out / f"{name}.json"
        bin_path.write_bytes(packed)
        json_path.write_text(
            json.dumps(
                {"header": header, "payloadHex": payload.hex(), "packedHex": packed.hex()},
                indent=2,
            )
        )
        written.extend([bin_path, json_path])
    return written

h264_sps_reorder_info(annexb)

Parse the SPS of an Annex B stream and return its reorder-relevant fields.

Keys: profile_idc, level_idc, vui_present, bitstream_restriction_flag, max_num_reorder_frames (None unless the restriction is present), max_dec_frame_buffering.

Why this matters: a WebCodecs hardware VideoDecoder that does not see max_num_reorder_frames=0 assumes worst-case reordering and buffers up to the level's DPB size before its first output. Under the session's small max_inflight that starves and the canvas silently freezes (no error). So the encoder must signal zero reordering.

Source code in src/pdum/rfb/testing.py 
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
def h264_sps_reorder_info(annexb: bytes) -> dict:
    """Parse the SPS of an Annex B stream and return its reorder-relevant fields.

    Keys: ``profile_idc``, ``level_idc``, ``vui_present``, ``bitstream_restriction_flag``,
    ``max_num_reorder_frames`` (``None`` unless the restriction is present),
    ``max_dec_frame_buffering``.

    Why this matters: a WebCodecs **hardware** ``VideoDecoder`` that does not see
    ``max_num_reorder_frames=0`` assumes worst-case reordering and buffers up to the level's
    DPB size before its first output. Under the session's small ``max_inflight`` that starves
    and the canvas silently freezes (no error). So the encoder must signal zero reordering.
    """
    sps = next((body for t, body in parse_nal_units(annexb) if t == 7), None)
    if sps is None:
        raise ValueError("no SPS (NAL type 7) in stream")
    b = _BitReader(_unescape_rbsp(sps[1:]))  # drop the NAL header byte
    profile_idc = b.u(8)
    b.u(8)  # constraint_set flags + reserved
    level_idc = b.u(8)
    b.ue()  # seq_parameter_set_id
    if profile_idc in (100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135):
        if b.ue() == 3:  # chroma_format_idc
            b.u1()  # separate_colour_plane
        b.ue()  # bit_depth_luma_minus8
        b.ue()  # bit_depth_chroma_minus8
        b.u1()  # qpprime_y_zero_transform_bypass
        if b.u1():  # seq_scaling_matrix_present (not emitted by baseline/main/high here)
            raise ValueError("scaling matrix present; parser not needed for this path")
    b.ue()  # log2_max_frame_num_minus4
    poc_type = b.ue()
    if poc_type == 0:
        b.ue()  # log2_max_pic_order_cnt_lsb_minus4
    elif poc_type == 1:
        b.u1()
        b.se()
        b.se()
        for _ in range(b.ue()):
            b.se()
    b.ue()  # max_num_ref_frames
    b.u1()  # gaps_in_frame_num_value_allowed
    b.ue()  # pic_width_in_mbs_minus1
    b.ue()  # pic_height_in_map_units_minus1
    if not b.u1():  # frame_mbs_only_flag
        b.u1()  # mb_adaptive_frame_field
    b.u1()  # direct_8x8_inference
    if b.u1():  # frame_cropping
        b.ue()
        b.ue()
        b.ue()
        b.ue()
    info = {
        "profile_idc": profile_idc,
        "level_idc": level_idc,
        "vui_present": bool(b.u1()),
        "bitstream_restriction_flag": None,
        "max_num_reorder_frames": None,
        "max_dec_frame_buffering": None,
    }
    if not info["vui_present"]:
        return info
    if b.u1() and b.u(8) == 255:  # aspect_ratio_info_present, Extended_SAR
        b.u(16)
        b.u(16)
    if b.u1():  # overscan_info_present
        b.u1()
    if b.u1():  # video_signal_type_present
        b.u(3)
        b.u1()
        if b.u1():  # colour_description_present
            b.u(24)
    if b.u1():  # chroma_loc_info_present
        b.ue()
        b.ue()
    if b.u1():  # timing_info_present
        b.u(32)
        b.u(32)
        b.u1()
    nal_hrd = b.u1()
    if nal_hrd:
        _skip_hrd(b)
    vcl_hrd = b.u1()
    if vcl_hrd:
        _skip_hrd(b)
    if nal_hrd or vcl_hrd:
        b.u1()  # low_delay_hrd
    b.u1()  # pic_struct_present
    if b.u1():  # bitstream_restriction_flag
        info["bitstream_restriction_flag"] = True
        b.u1()  # motion_vectors_over_pic_boundaries
        b.ue()  # max_bytes_per_pic_denom
        b.ue()  # max_bits_per_mb_denom
        b.ue()  # log2_max_mv_length_horizontal
        b.ue()  # log2_max_mv_length_vertical
        info["max_num_reorder_frames"] = b.ue()
        info["max_dec_frame_buffering"] = b.ue()
    else:
        info["bitstream_restriction_flag"] = False
    return info

has_sps_pps_idr(annexb)

True if the stream contains SPS (7), PPS (8) and an IDR slice (5).

Source code in src/pdum/rfb/testing.py 
280
281
282
283
def has_sps_pps_idr(annexb: bytes) -> bool:
    """True if the stream contains SPS (7), PPS (8) and an IDR slice (5)."""
    types = nal_types(annexb)
    return {7, 8, 5}.issubset(types)

loopback_server(handler, *, host='127.0.0.1', port=0) async

Run a real websockets server for one integration test.

Yields (host, port) of the listening server; the OS assigns a free port when port is 0.

Source code in src/pdum/rfb/testing.py 
467
468
469
470
471
472
473
474
475
476
477
478
479
@contextlib.asynccontextmanager
async def loopback_server(handler, *, host: str = "127.0.0.1", port: int = 0):
    """Run a real ``websockets`` server for one integration test.

    Yields ``(host, port)`` of the listening server; the OS assigns a free port
    when ``port`` is 0.
    """
    import websockets.asyncio.server

    async with websockets.asyncio.server.serve(handler, host, port) as server:
        sock = next(iter(server.sockets))
        bound_host, bound_port = sock.getsockname()[:2]
        yield bound_host, bound_port

nal_types(annexb)

Return the set of NAL unit types present in an Annex B stream.

Source code in src/pdum/rfb/testing.py 
275
276
277
def nal_types(annexb: bytes) -> set[int]:
    """Return the set of NAL unit types present in an Annex B stream."""
    return {t for t, _ in parse_nal_units(annexb)}

parse_nal_units(annexb)

Split an Annex B byte stream into (nal_type, body) tuples.

Source code in src/pdum/rfb/testing.py 
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
def parse_nal_units(annexb: bytes) -> list[tuple[int, bytes]]:
    """Split an Annex B byte stream into ``(nal_type, body)`` tuples."""
    units: list[tuple[int, bytes]] = []
    data = bytes(annexb)
    starts: list[int] = []
    i = 0
    n = len(data)
    while i < n - 3:
        if data[i] == 0 and data[i + 1] == 0:
            if data[i + 2] == 1:
                starts.append((i, 3))
                i += 3
                continue
            if data[i + 2] == 0 and i < n - 4 and data[i + 3] == 1:
                starts.append((i, 4))
                i += 4
                continue
        i += 1
    for idx, (pos, sc_len) in enumerate(starts):
        body_start = pos + sc_len
        body_end = starts[idx + 1][0] if idx + 1 < len(starts) else n
        body = data[body_start:body_end]
        if body:
            nal_type = body[0] & 0x1F
            units.append((nal_type, body))
    return units

render_pattern(name, seq, width, height)

Render any named pattern (used by the benchmark harness).

Source code in src/pdum/rfb/testing.py 
115
116
117
118
119
def render_pattern(name: str, seq: int, width: int, height: int) -> np.ndarray:
    """Render any named pattern (used by the benchmark harness)."""
    if name not in _PATTERNS:
        raise ValueError(f"unknown pattern {name!r}; choose from {sorted(_PATTERNS)}")
    return _PATTERNS[name](seq, width, height)

render_test_pattern(seq, width, height)

Canonical deterministic RGB test pattern (shared with the JS e2e tests).

Four quadrants, each a flat color that cycles by seq so consecutive frames differ (exercising inter-frame compression) while interior pixels stay flat (robust to lossy decoding). The browser test recomputes the expected quadrant colors for the displayed seq and compares.

Source code in src/pdum/rfb/testing.py 
37
38
39
40
41
42
43
44
45
46
47
48
49
50
def render_test_pattern(seq: int, width: int, height: int) -> np.ndarray:
    """Canonical deterministic RGB test pattern (shared with the JS e2e tests).

    Four quadrants, each a flat color that cycles by ``seq`` so consecutive
    frames differ (exercising inter-frame compression) while interior pixels
    stay flat (robust to lossy decoding). The browser test recomputes the
    expected quadrant colors for the displayed ``seq`` and compares.
    """
    arr = np.empty((height, width, 3), dtype=np.uint8)
    hw, hh = width // 2, height // 2
    quadrants = [(0, hh, 0, hw), (0, hh, hw, width), (hh, height, 0, hw), (hh, height, hw, width)]
    for q, (y0, y1, x0, x1) in enumerate(quadrants):
        arr[y0:y1, x0:x1] = _QUADRANT_COLORS[(q + seq) % 4]
    return arr

starts_with_start_code(data)

True if data begins with an Annex B start code (not AVCC length).

Source code in src/pdum/rfb/testing.py 
286
287
288
def starts_with_start_code(data: bytes) -> bool:
    """True if ``data`` begins with an Annex B start code (not AVCC length)."""
    return data[:4] == b"\x00\x00\x00\x01" or data[:3] == b"\x00\x00\x01"

transport

Transport seam between :class:~pdum.rfb.session.RfbSession and the socket.

The session is deliberately ignorant of how bytes move: it only needs to await send(...) and async for over inbound messages. :class:Channel captures exactly that surface, and :class:WebSocketTransport adapts a websockets connection to it.

This is the additive transport seam: the Starlette/ASGI WebSocket adapter (mapping WebSocketDisconnect onto the ConnectionClosed type the session already catches) drops in here, with no change to the session, encoders, or sources.

Channel

Bases: Protocol

The minimal duplex byte/text channel the session drives.

Source code in src/pdum/rfb/transport.py 
18
19
20
21
22
23
24
25
26
27
28
@runtime_checkable
class Channel(Protocol):
    """The minimal duplex byte/text channel the session drives."""

    async def send(self, data: bytes | str) -> None:
        """Send one binary payload or one text control message."""
        ...

    def __aiter__(self) -> AsyncIterator[bytes | str]:
        """Asynchronously iterate inbound messages (``bytes`` or ``str``)."""
        ...

__aiter__()

Asynchronously iterate inbound messages (bytes or str).

Source code in src/pdum/rfb/transport.py 
26
27
28
def __aiter__(self) -> AsyncIterator[bytes | str]:
    """Asynchronously iterate inbound messages (``bytes`` or ``str``)."""
    ...

send(data) async

Send one binary payload or one text control message.

Source code in src/pdum/rfb/transport.py 
22
23
24
async def send(self, data: bytes | str) -> None:
    """Send one binary payload or one text control message."""
    ...

WebSocketTransport

Adapt a websockets server connection to the :class:Channel surface.

A raw websockets connection already satisfies :class:Channel; this thin wrapper exists as the documented seam (and one place to translate disconnect semantics for non-websockets transports later).

Source code in src/pdum/rfb/transport.py 
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
class WebSocketTransport:
    """Adapt a ``websockets`` server connection to the :class:`Channel` surface.

    A raw ``websockets`` connection already satisfies :class:`Channel`; this thin
    wrapper exists as the documented seam (and one place to translate disconnect
    semantics for non-``websockets`` transports later).
    """

    __slots__ = ("_ws",)

    def __init__(self, ws: Any) -> None:
        self._ws = ws

    async def send(self, data: bytes | str) -> None:
        await self._ws.send(data)

    def __aiter__(self) -> AsyncIterator[bytes | str]:
        return self._ws.__aiter__()

    async def close(self, code: int = 1000, reason: str = "") -> None:
        await self._ws.close(code, reason)

types

Core data types and protocols for the remote framebuffer.

This module is intentionally dependency-free (no Pillow / PyAV / websockets) so that import pdum.rfb.types is always cheap and safe, even in environments that only need the type definitions.

The design follows three independent concerns (see the implementation guide):

Frame source -> Encoder backend -> Transport backend

ColorSpace dataclass

A small, explicit display-referred color descriptor for a frame/stream.

Mirrors the WebCodecs VideoColorSpace fields the browser consumes directly (primaries gamut, transfer function, matrix RGB-vs-YUV coupling) plus a full_range flag and a bit_depth for the HDR future. Two presets ship as first-class — :data:SRGB (the implicit default) and :data:DISPLAY_P3 (Apple wide-gamut SDR, 8-bit); bt2020/pq/hlg are expressible (HDR is designed-for) but not yet wired through a 10-bit pipeline.

The library tags color; it does not convert. The upstream renderer is responsible for producing pixels already in the declared space.

Source code in src/pdum/rfb/types.py 
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
@dataclass(slots=True, frozen=True)
class ColorSpace:
    """A small, explicit display-referred color descriptor for a frame/stream.

    Mirrors the WebCodecs ``VideoColorSpace`` fields the browser consumes directly
    (``primaries`` gamut, ``transfer`` function, ``matrix`` RGB-vs-YUV coupling) plus a
    ``full_range`` flag and a ``bit_depth`` for the HDR future. Two presets ship as
    first-class — :data:`SRGB` (the implicit default) and :data:`DISPLAY_P3` (Apple
    wide-gamut SDR, 8-bit); ``bt2020``/``pq``/``hlg`` are expressible (HDR is designed-for)
    but not yet wired through a 10-bit pipeline.

    The library **tags** color; it does not convert. The upstream renderer is responsible
    for producing pixels already in the declared space.
    """

    primaries: Primaries = "bt709"
    transfer: Transfer = "srgb"
    matrix: ColorMatrix = "rgb"
    full_range: bool = True
    bit_depth: int = 8

    def to_dict(self) -> dict:
        """The wire/JSON form (as carried on frame headers and the ``config`` message)."""
        return asdict(self)

to_dict()

The wire/JSON form (as carried on frame headers and the config message).

Source code in src/pdum/rfb/types.py 
50
51
52
def to_dict(self) -> dict:
    """The wire/JSON form (as carried on frame headers and the ``config`` message)."""
    return asdict(self)

DownscaleHint dataclass

A server-driven adaptive resolution hint delivered through poll_events().

In the push model the publisher owns the framebuffer size, so the adaptive controller cannot resize the stream itself — its deepest congestion lever is to suggest a smaller render resolution and let the render loop honor it. Enabled by serve(adaptive=True), the controller emits one of these under sustained pressure (bitrate, fps, and in-flight already at their floors) and again on recovery (with scale climbing back toward 1.0); the :class:~pdum.rfb.display.Display fans the aggregate across viewers (the most-congested viewer wins) into the same queue :meth:~pdum.rfb.display.Display.poll_events drains, tagged as this distinct type so it is easy to tell apart from an :class:InputEvent::

for ev in display.poll_events():
    if isinstance(ev, rfb.DownscaleHint):
        w, h = ev.width, ev.height        # render/publish at this size
    else:
        state = update(state, ev.event)   # ordinary input event

Honoring the hint is opt-in: publish a frame of the suggested size (the per-viewer encoder rebuilds and the browser re-configure()s exactly as for any other resize) or ignore it entirely — the stream keeps working either way.

Parameters:

Name Type Description Default
scale float

The recommended render scale relative to full resolution, in (0, 1] (1.0 = full res). Absolute, not incremental, so applying it never compounds: round(base_w * scale) always sizes off your native resolution.

 required
width int

A convenience suggested size — the display's initial (base) dimensions scaled by scale and rounded to even values (H.264 / NV12 need even dimensions).

 required
height int

A convenience suggested size — the display's initial (base) dimensions scaled by scale and rounded to even values (H.264 / NV12 need even dimensions).

 required
received_us int

Monotonic microseconds (relative to the display's start) when the hint was emitted.

 required
Source code in src/pdum/rfb/types.py 
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
@dataclass(slots=True, frozen=True)
class DownscaleHint:
    """A server-driven adaptive **resolution** hint delivered through ``poll_events()``.

    In the push model the publisher owns the framebuffer size, so the adaptive
    controller cannot resize the stream itself — its deepest congestion lever is to
    *suggest* a smaller render resolution and let the render loop honor it. Enabled by
    ``serve(adaptive=True)``, the controller emits one of these under sustained pressure
    (bitrate, fps, and in-flight already at their floors) and again on recovery (with
    ``scale`` climbing back toward ``1.0``); the :class:`~pdum.rfb.display.Display` fans
    the aggregate across viewers (the most-congested viewer wins) into the same queue
    :meth:`~pdum.rfb.display.Display.poll_events` drains, tagged as this distinct type so
    it is easy to tell apart from an :class:`InputEvent`::

        for ev in display.poll_events():
            if isinstance(ev, rfb.DownscaleHint):
                w, h = ev.width, ev.height        # render/publish at this size
            else:
                state = update(state, ev.event)   # ordinary input event

    Honoring the hint is opt-in: publish a frame of the suggested size (the per-viewer
    encoder rebuilds and the browser re-``configure()``s exactly as for any other
    resize) or ignore it entirely — the stream keeps working either way.

    Parameters
    ----------
    scale:
        The recommended render scale relative to full resolution, in ``(0, 1]``
        (``1.0`` = full res). **Absolute**, not incremental, so applying it never
        compounds: ``round(base_w * scale)`` always sizes off your native resolution.
    width, height:
        A convenience suggested size — the display's initial (base) dimensions scaled
        by ``scale`` and rounded to even values (H.264 / NV12 need even dimensions).
    received_us:
        Monotonic microseconds (relative to the display's start) when the hint was
        emitted.
    """

    scale: float
    width: int
    height: int
    received_us: int

EncodedPayload dataclass

A single encoded payload ready to be put on the wire.

One image is one payload (always a keyframe). One encoded video access unit is one payload; keyframe marks IDR access units.

Source code in src/pdum/rfb/types.py 
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
@dataclass(slots=True)
class EncodedPayload:
    """A single encoded payload ready to be put on the wire.

    One image is one payload (always a keyframe). One encoded video access unit
    is one payload; ``keyframe`` marks IDR access units.
    """

    seq: int
    kind: EncodedKind
    timestamp_us: int
    payload: bytes
    width: int
    height: int
    mime: str | None = None  # e.g. "image/jpeg", "image/png"
    codec: str | None = None  # e.g. "avc1.42E01F"
    keyframe: bool = False
    duration_us: int | None = None
    metadata: dict[str, Any] | None = None
    # Render-side descriptors carried from the source RawFrame onto the wire header. The
    # session stamps these from the frame after encode (so no encoder needs to know about
    # them); ``header_for`` emits them when non-default. See RawFrame.pixel_ratio / color.
    pixel_ratio: float = 1.0
    color: dict | None = None

EncoderBackend

Bases: Protocol

Turns raw frames into encoded payloads.

Source code in src/pdum/rfb/types.py 
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
@runtime_checkable
class EncoderBackend(Protocol):
    """Turns raw frames into encoded payloads."""

    def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
        """Encode a single frame, returning zero or more payloads."""
        ...

    def flush(self) -> list[EncodedPayload]:
        """Drain any buffered payloads from the encoder."""
        ...

    def close(self) -> None:
        """Release encoder resources."""
        ...

close()

Release encoder resources.

Source code in src/pdum/rfb/types.py 
231
232
233
def close(self) -> None:
    """Release encoder resources."""
    ...

encode(frame, *, force_keyframe=False)

Encode a single frame, returning zero or more payloads.

Source code in src/pdum/rfb/types.py 
223
224
225
def encode(self, frame: RawFrame, *, force_keyframe: bool = False) -> list[EncodedPayload]:
    """Encode a single frame, returning zero or more payloads."""
    ...

flush()

Drain any buffered payloads from the encoder.

Source code in src/pdum/rfb/types.py 
227
228
229
def flush(self) -> list[EncodedPayload]:
    """Drain any buffered payloads from the encoder."""
    ...

FrameSource

Bases: Protocol

Produces raw frames and consumes user-input events.

Internal SPI: the session pulls frames through this shape. Applications no longer implement it directly — they push frames via :class:~pdum.rfb.display.Display.

Source code in src/pdum/rfb/types.py 
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
@runtime_checkable
class FrameSource(Protocol):
    """Produces raw frames and consumes user-input events.

    Internal SPI: the session pulls frames through this shape. Applications no
    longer implement it directly — they push frames via
    :class:`~pdum.rfb.display.Display`.
    """

    async def next_frame(self) -> RawFrame:
        """Return the next frame to encode (may block / pace to a target fps)."""
        ...

    async def handle_event(self, event: EventDict) -> None:
        """Handle a normalized user-input event from the client."""
        ...

handle_event(event) async

Handle a normalized user-input event from the client.

Source code in src/pdum/rfb/types.py 
214
215
216
async def handle_event(self, event: EventDict) -> None:
    """Handle a normalized user-input event from the client."""
    ...

next_frame() async

Return the next frame to encode (may block / pace to a target fps).

Source code in src/pdum/rfb/types.py 
210
211
212
async def next_frame(self) -> RawFrame:
    """Return the next frame to encode (may block / pace to a target fps)."""
    ...

InputEvent dataclass

A normalized user-input event delivered to the application.

Produced by :class:~pdum.rfb.display.Display as it fans connected clients' input into a single stream the application drains with :meth:~pdum.rfb.display.Display.poll_events.

Parameters:

Name Type Description Default
client_id str

Opaque per-connection identifier, so several viewers on one display can be told apart (e.g. for multi-user coordination).

 required
principal Any | None

Whatever the authenticate hook returned for this connection (an application-defined identity), or None when auth is disabled.

 required
event EventDict

The raw normalized event dict ({"type": "pointer_move", "x": ..., ...}).

 required
received_us int

Monotonic microseconds (relative to the display's start) when the event was received.

 required
Source code in src/pdum/rfb/types.py 
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
@dataclass(slots=True)
class InputEvent:
    """A normalized user-input event delivered to the application.

    Produced by :class:`~pdum.rfb.display.Display` as it fans connected clients'
    input into a single stream the application drains with
    :meth:`~pdum.rfb.display.Display.poll_events`.

    Parameters
    ----------
    client_id:
        Opaque per-connection identifier, so several viewers on one display can be
        told apart (e.g. for multi-user coordination).
    principal:
        Whatever the ``authenticate`` hook returned for this connection (an
        application-defined identity), or ``None`` when auth is disabled.
    event:
        The raw normalized event dict (``{"type": "pointer_move", "x": ..., ...}``).
    received_us:
        Monotonic microseconds (relative to the display's start) when the event
        was received.
    """

    client_id: str
    principal: Any | None
    event: EventDict
    received_us: int

RawFrame dataclass

A single raw frame produced by a :class:FrameSource.

Parameters:

Name Type Description Default
seq int

Monotonically increasing frame sequence number.

 required
width int

Frame dimensions in pixels.

 required
height int

Frame dimensions in pixels.

 required
timestamp_us int

Capture/render timestamp in microseconds.

 required
pixel_format PixelFormat

Layout of data (e.g. "rgb24", "rgba8", "nv12").

 required
memory MemoryKind

Where data lives ("cpu", "cuda" or "opengl").

 required
data Any

The pixel payload. For CPU frames this is a numpy.ndarray of uint8; for rgb24 the shape is (H, W, 3) and for rgba8 it is (H, W, 4).

 required
pixel_ratio float

Render-side device-pixels-per-logical-pixel of this frame (default 1.0). Display intent, not a resample instruction: the pixels are delivered as-is; the client uses it to size the frame in logical space for fit, and echoes it on input events so a publisher rendering in logical coordinates can divide it out.

 1.0
color dict | None

Optional color descriptor (dict form of a :class:ColorSpace); None means sRGB. The renderer must produce pixels in the declared space (no conversion here).

 None
Source code in src/pdum/rfb/types.py 
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
@dataclass(slots=True)
class RawFrame:
    """A single raw frame produced by a :class:`FrameSource`.

    Parameters
    ----------
    seq:
        Monotonically increasing frame sequence number.
    width, height:
        Frame dimensions in pixels.
    timestamp_us:
        Capture/render timestamp in microseconds.
    pixel_format:
        Layout of ``data`` (e.g. ``"rgb24"``, ``"rgba8"``, ``"nv12"``).
    memory:
        Where ``data`` lives (``"cpu"``, ``"cuda"`` or ``"opengl"``).
    data:
        The pixel payload. For CPU frames this is a ``numpy.ndarray`` of
        ``uint8``; for ``rgb24`` the shape is ``(H, W, 3)`` and for ``rgba8``
        it is ``(H, W, 4)``.
    pixel_ratio:
        Render-side device-pixels-per-logical-pixel of this frame (default ``1.0``).
        Display intent, not a resample instruction: the pixels are delivered as-is; the
        client uses it to size the frame in *logical* space for fit, and echoes it on
        input events so a publisher rendering in logical coordinates can divide it out.
    color:
        Optional color descriptor (``dict`` form of a :class:`ColorSpace`); ``None`` means
        sRGB. The renderer must produce pixels **in** the declared space (no conversion here).
    """

    seq: int
    width: int
    height: int
    timestamp_us: int
    pixel_format: PixelFormat
    memory: MemoryKind
    data: Any
    pixel_ratio: float = 1.0
    color: dict | None = None