{"templateId":"markdown","sharedDataIds":{"sidebar":"sidebar-sidebars.yaml"},"props":{"metadata":{"markdoc":{"tagList":[]},"type":"markdown"},"seo":{"title":"Realtime Video Background Removal API","siteUrl":"https://docs.bria.ai","llmstxt":{"hide":false,"title":"Bria AI API","description":"bria.ai helps you build products and workflows by delivering controllable, easy-to-integrate and safe generative visual AI capabilities.","details":{"path":"_llmstxt-details.md"},"sections":[{"title":"Integrations","description":"MCP server, Claude Skill, and Python SDK.","includeFiles":["MCP-authentication.md","integration-methods/bria-skill.md","integration-methods/python-sdk.md"],"excludeFiles":[]},{"title":"Getting Started","description":"Authentication, API overview, and platform concepts.","includeFiles":["index.md","products-overview.md","safety.md","vgl.md","best-practices-overview.md"],"excludeFiles":[]},{"title":"Image Generation","description":"Generate images using text prompts or structured JSON via FIBO models.","includeFiles":["image-generation.md","image-generation.yaml"],"excludeFiles":[]},{"title":"Image Editing","description":"Edit, transform, and enhance images with 20+ specialized endpoints.","includeFiles":["image-editing.md","image-editing.yaml"],"excludeFiles":[]},{"title":"Video Editing","description":"Remove backgrounds (REST async or real-time streaming), erase objects, upscale, and generate masks for video.","includeFiles":["video-editing.md","local-video-upload-service.md","video-editing.yaml","streaming-rmbg.md"],"excludeFiles":[]},{"title":"Product Shot Editing","description":"SKU packshots, lifestyle scenes, and automotive product imagery.","includeFiles":["product-shot-editing.md","product-shot-editing.yaml"],"excludeFiles":[]},{"title":"Optional","description":"Tailored generation, ads, image onboarding, and attribution.","includeFiles":["tailored-generation.md","campaign-generation.md","image-onboarding.md","bria-attribution-service.md","ad-generation.md","tailored-generation.yaml","campaign-generation.yaml","image-onboarding.yaml","bria-attribution-service.yaml","ad-generation.yaml"],"excludeFiles":[]}],"excludeFiles":["_llmstxt-details.md","_partials.md","CHANGELOG.md"]}},"dynamicMarkdocComponents":[],"compilationErrors":[],"ast":{"$$mdtype":"Tag","name":"article","attributes":{},"children":[{"$$mdtype":"Tag","name":"Heading","attributes":{"level":1,"id":"realtime-video-background-removal-api","__idx":0},"children":["Realtime Video Background Removal API"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Remove or change backgrounds from video streams in real-time over a persistent WebSocket connection. Send JPEG frames, receive grayscale JPEG masks (0 = background, 1 = foreground), and composite client-side."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":[{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://platform.bria.ai/video-editing/video-streaming-remove-background/sandbox"},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Try out this capability in Bria's sandbox"]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Working end-to-end client (browser): ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/bria-ai/streaming-rmbg-example"},"children":["streaming-rmbg-example"]}]},{"$$mdtype":"Tag","name":"blockquote","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["💡 ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Processing finite video files instead?"]}," For batch workflows that need audio preservation, codec/container selection, or full editing capabilities, use the REST ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"/video-editing#operation/remove-background"},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["/remove_background"]}]}," endpoint."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"performance","__idx":1},"children":["Performance"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["US-based clients can achieve ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["~24 FPS"]}," real-time throughput with ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["<100ms RTT"]}," (~74 ms p50, ~97 ms p95)."]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Hitting the 24 FPS ceiling typically requires client-side optimizations — see ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#client-side-compositing--optimization"},"children":["Client-Side Compositing & Optimization"]}," below."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"connection","__idx":2},"children":["Connection"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"wss://streaming.prod.bria-api.com\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"authentication","__idx":3},"children":["Authentication"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Authenticate using either an API token ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["or"]}," an OAuth access token — provide only one per connection."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"api-token","__idx":4},"children":["API token"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"wss://streaming.prod.bria-api.com?api_token=<YOUR_API_TOKEN>\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"oauth","__idx":5},"children":["OAuth"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"wss://streaming.prod.bria-api.com?oauth=<YOUR_OAUTH_ACCESS_TOKEN>\n"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"message-format","__idx":6},"children":["Message format"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The protocol uses two message types over the same connection:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Binary messages"]}," — video/audio frame data. 24-byte header + JPEG payload."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["JSON messages"]}," — session control, errors, and debug telemetry."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"binary-frame-header-24-bytes","__idx":7},"children":["Binary frame header (24 bytes)"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Offset"},"children":["Offset"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Size"},"children":["Size"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Field"},"children":["Field"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Type"},"children":["Type"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Value"},"children":["Value"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["0–3"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["4 B"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Magic"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["string"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["BRIA"]}," (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["0x42524941"]},")"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["4"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1 B"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Version"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["uint8"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["3"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["5"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1 B"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["App ID"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["uint8"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["1"]}," (Remove Background)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["6"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1 B"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Media Type"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["uint8"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["1"]},"=Video, ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["2"]},"=Audio*"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["7"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1 B"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Codec"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["uint8"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["1"]},"=JPEG"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["8–15"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["8 B"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Frame ID"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["uint64"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Big-endian frame ID†"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["16–23"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["8 B"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["PTS"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["int64"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Big-endian timestamp‡"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["24+"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["var"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Payload"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["bytes"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["JPEG frame (client→server) or mask (server→client)"]}]}]}]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["* Video is processed and a mask is returned; audio passes through untouched.",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]},"† ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Frame ID"]}," is echoed by the server on the corresponding mask, so the client can pair masks with their source frames.",{"$$mdtype":"Tag","name":"br","attributes":{},"children":[]},"‡ ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["PTS"]}," is opaque to the server and echoed back unchanged for client-side timing logic."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"constants","__idx":8},"children":["Constants"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Constant"},"children":["Constant"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Value"},"children":["Value"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["MAGIC"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["BRIA"]}," (",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["0x42524941"]},")"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["VERSION"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["3"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["APP_RMBG"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["1"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["MEDIA_VIDEO"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["1"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["MEDIA_AUDIO"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["2"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["CODEC_JPEG"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["1"]}]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["HEADER_SIZE"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["24"]}]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"client--server","__idx":9},"children":["Client → Server"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"video-frame-binary","__idx":10},"children":["Video frame (binary)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"header":{"controls":{"copy":{}}},"source":"[24-byte header][JPEG payload]\n"},"children":[]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Encode each captured frame as JPEG, prepend the header, and send. Use a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["monotonically increasing ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["frame_id"]}]}," so the server can echo it back on the corresponding mask."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"stop-session-json","__idx":11},"children":["Stop session (JSON)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{ \"type\": \"stop\" }\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"debug-mode-json","__idx":12},"children":["Debug mode (JSON)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Enables per-frame timing telemetry on the server's responses."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{ \"type\": \"debug\", \"enabled\": true }\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{ \"type\": \"debug\", \"enabled\": false }\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"server--client","__idx":13},"children":["Server → Client"]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"mask-frame-binary","__idx":14},"children":["Mask frame (binary)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Same 24-byte header layout as the inbound video frame. The payload is a ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["grayscale JPEG mask"]},", with ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["frame_id"]}," echoed from the source frame."]},{"$$mdtype":"Tag","name":"blockquote","attributes":{},"children":[{"$$mdtype":"Tag","name":"p","attributes":{},"children":["⚠️ The mask is ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["not RGBA"]}," and contains no color from the original frame. Use it as alpha against the matching source frame and composite client-side. See ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"#client-side-compositing--optimization"},"children":["Client-Side Compositing & Optimization"]},"."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"error-json","__idx":15},"children":["Error (JSON)"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{ \"type\": \"error\", \"message\": \"...\" }\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"frame-timing-json-debug-only","__idx":16},"children":["Frame timing (JSON, debug only)"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["Returned only when debug mode is enabled."]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"json","header":{"controls":{"copy":{}}},"source":"{\n  \"type\": \"frame_timing\",\n  \"frame_id\": 4,\n  \"frames_dropped_since_last\": 0,\n  \"server_ms\": {\n    \"decode_jpeg\": 0.5,\n    \"pre\": 0.17,\n    \"infer\": 38.83,\n    \"post\": 0.11,\n    \"encode_mask\": 0.22,\n    \"engine_total\": 40.22,\n    \"session_total\": 41.34\n  },\n  \"bytes_in\": 60881,\n  \"bytes_out\": 23254\n}\n","lang":"json"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"websocket-close-codes","__idx":17},"children":["WebSocket close codes"]},{"$$mdtype":"Tag","name":"div","attributes":{"className":"md-table-wrapper"},"children":[{"$$mdtype":"Tag","name":"table","attributes":{"className":"md"},"children":[{"$$mdtype":"Tag","name":"thead","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Code"},"children":["Code"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Reason"},"children":["Reason"]},{"$$mdtype":"Tag","name":"th","attributes":{"data-label":"Trigger"},"children":["Trigger"]}]}]},{"$$mdtype":"Tag","name":"tbody","attributes":{},"children":[{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1008"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["unauthorized"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Authentication failed or invalid credentials"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["1013"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["capacity exceeded, please try again later"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Service is temporarily overloaded"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["4003"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["session limit reached"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["Session duration limit reached (plan restriction)"]}]},{"$$mdtype":"Tag","name":"tr","attributes":{},"children":[{"$$mdtype":"Tag","name":"td","attributes":{},"children":["4008"]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":[{"$$mdtype":"Tag","name":"code","attributes":{},"children":["session timeout"]}]},{"$$mdtype":"Tag","name":"td","attributes":{},"children":["No media frames received within allowed time"]}]}]}]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"client-side-compositing--optimization","__idx":18},"children":["Client-side compositing & optimization"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["The API returns ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["mask JPEGs only"]},". Your integration is responsible for:"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Producing video frames as JPEG with monotonically increasing ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["frame_id"]}," values."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Keeping each source frame (or its JPEG) until the matching mask returns."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Compositing mask × foreground (and any background) on your side. Backgrounds are ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["never inferred by the server"]}," — solid colors, images, or video behind the subject are entirely client-side."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"algorithm-patterns","__idx":19},"children":["Algorithm patterns"]},{"$$mdtype":"Tag","name":"ol","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Capture and encode on a steady clock"]}," — Typically once per display frame (or your pipeline's tick). Run JPEG encode and heavy blending ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["off the UI thread"]}," (worker, native module, GPU queue) so you do not block interaction or capture."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Latest-frame-wins (unsent queue)"]}," — If encoding or the network falls behind, ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["drop older frames that were never sent"]}," and keep only the ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["newest"]}," ready-to-send JPEG. A deep FIFO of unsent frames adds latency without improving quality."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Bounded in-flight (AIMD-style window)"]}," — Cap how many frames you have ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["sent but not yet received a mask for"]},". Grow that cap slowly when round-trip times are stable, ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["shrink it"]}," when RTT spikes or when sends time out (\"stale\"). This self-throttles bandwidth and protects the server and your own memory."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Same binary envelope both ways"]}," — Outbound video and inbound masks both use the same 24-byte header + JPEG payload; parse inbound ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["frame_id"]}," with the same rules you used when packing outbound frames."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":[{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Mask handling"]}," — Decode the mask JPEG to a bitmap (single-channel / grayscale weights after compression). ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["Look up the source frame for that ",{"$$mdtype":"Tag","name":"code","attributes":{},"children":["frame_id"]}]},", multiply foreground by mask alpha, add background where you want it, output RGBA or premultiplied BGRA as your renderer needs. Optionally ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["serialize or cap"]}," compositor work so a burst of replies cannot stall the app."]}]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"practical-tuning","__idx":20},"children":["Practical tuning"]},{"$$mdtype":"Tag","name":"ul","attributes":{},"children":[{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Reduce ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["frame size"]}," and/or ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["JPEG quality"]}," to lower CPU and uplink bytes."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Prefer ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["GPU compositing"]}," for full-resolution blends; separating ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["encode"]}," and ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["composite"]}," work avoids one stage blocking the other."]},{"$$mdtype":"Tag","name":"li","attributes":{},"children":["Throughput will track ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["network RTT"]},", ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["server capacity"]},", and ",{"$$mdtype":"Tag","name":"strong","attributes":{},"children":["how fast you can produce JPEGs"]}," — measure masks/sec vs. composite time separately if you instrument both."]}]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A browser app implementing all of the patterns above is available as a reference: ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/bria-ai/streaming-rmbg-example"},"children":["streaming-rmbg-example"]},"."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":2,"id":"examples","__idx":21},"children":["Examples"]},{"$$mdtype":"Tag","name":"p","attributes":{},"children":["A complete end-to-end client (capture, send, receive, composite) lives in ",{"$$mdtype":"Tag","name":"MarkdownLink","attributes":{"href":"https://github.com/bria-ai/streaming-rmbg-example"},"children":["streaming-rmbg-example"]},". The snippets below illustrate the wire protocol."]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"javascript-connect-and-send","__idx":22},"children":["JavaScript: connect and send"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"js","header":{"controls":{"copy":{}}},"source":"const HEADER_SIZE = 24;\nconst VERSION = 3;\nconst APP = 1;\nconst MEDIA_VIDEO = 1;\nconst CODEC_JPEG = 1;\n\nfunction buildWsUrl(serverUrl, auth) {\n  const url = new URL(serverUrl);\n\n  if (auth.apiToken) {\n    url.searchParams.set(\"api_token\", auth.apiToken);\n  } else if (auth.oauth) {\n    url.searchParams.set(\"oauth\", auth.oauth);\n  } else {\n    throw new Error(\"Missing authentication\");\n  }\n\n  return url.toString();\n}\n\nfunction buildHeader(frameId, ptsUs) {\n  const buf = new ArrayBuffer(HEADER_SIZE);\n  const view = new DataView(buf);\n\n  view.setUint8(0, 0x42);\n  view.setUint8(1, 0x52);\n  view.setUint8(2, 0x49);\n  view.setUint8(3, 0x41);\n\n  view.setUint8(4, VERSION);\n  view.setUint8(5, APP);\n  view.setUint8(6, MEDIA_VIDEO);\n  view.setUint8(7, CODEC_JPEG);\n\n  view.setUint32(8, Math.floor(frameId / 0x100000000), false);\n  view.setUint32(12, frameId >>> 0, false);\n\n  const ptsHi = Math.floor(ptsUs / 0x100000000);\n  const ptsLo = ptsUs >>> 0;\n\n  view.setInt32(16, ptsHi, false);\n  view.setUint32(20, ptsLo, false);\n\n  return buf;\n}\n\nconst ws = new WebSocket(buildWsUrl(\n  \"wss://streaming.prod.bria-api.com\",\n  { apiToken: \"YOUR_TOKEN\" }\n));\n\nws.binaryType = \"arraybuffer\";\n\nws.onmessage = (event) => {\n  if (typeof event.data === \"string\") return;\n\n  const buf = event.data;\n  // Same 24-byte header as outbound; remainder is the mask JPEG.\n  const jpegMask = buf.slice(HEADER_SIZE);\n  // Parse frame_id from the header, load the matching source frame you kept for that id,\n  // decode both JPEGs, composite (foreground × mask + optional background).\n  // Reference: https://github.com/bria-ai/streaming-rmbg-example\n};\n","lang":"js"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"javascript-parse-mask-header","__idx":23},"children":["JavaScript: parse mask header"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"js","header":{"controls":{"copy":{}}},"source":"function unpackHeader(buffer) {\n  const view = new DataView(buffer);\n\n  if (\n    view.getUint8(0) !== 0x42 ||\n    view.getUint8(1) !== 0x52 ||\n    view.getUint8(2) !== 0x49 ||\n    view.getUint8(3) !== 0x41\n  ) return null;\n\n  return {\n    frameId:\n      view.getUint32(8, false) * 0x100000000 +\n      view.getUint32(12, false),\n    ptsUs:\n      view.getInt32(16, false) * 0x100000000 +\n      view.getUint32(20, false),\n    payload: buffer.slice(24),\n  };\n}\n","lang":"js"},"children":[]},{"$$mdtype":"Tag","name":"Heading","attributes":{"level":3,"id":"python-connect-and-send","__idx":24},"children":["Python: connect and send"]},{"$$mdtype":"Tag","name":"CodeBlock","attributes":{"data-language":"python","header":{"controls":{"copy":{}}},"source":"import struct\nimport asyncio\nimport websockets\n\nHEADER_SIZE = 24\n\ndef build_header(frame_id, pts_us):\n    buf = bytearray(HEADER_SIZE)\n\n    buf[0:4] = b\"BRIA\"\n    buf[4] = 3\n    buf[5] = 1\n    buf[6] = 1\n    buf[7] = 1\n\n    struct.pack_into(\">I\", buf, 8, frame_id >> 32)\n    struct.pack_into(\">I\", buf, 12, frame_id & 0xFFFFFFFF)\n\n    pts_hi = pts_us >> 32\n    pts_lo = pts_us & 0xFFFFFFFF\n\n    struct.pack_into(\">i\", buf, 16, pts_hi)\n    struct.pack_into(\">I\", buf, 20, pts_lo)\n\n    return bytes(buf)\n\ndef build_ws_url(server_url, api_token=None, oauth=None):\n    if bool(api_token) == bool(oauth):\n        raise ValueError(\"Provide exactly one auth method\")\n\n    sep = \"&\" if \"?\" in server_url else \"?\"\n    if api_token:\n        return f\"{server_url}{sep}api_token={api_token}\"\n    return f\"{server_url}{sep}oauth={oauth}\"\n\nasync def main():\n    uri = build_ws_url(\n        \"wss://streaming.prod.bria-api.com\",\n        api_token=\"YOUR_TOKEN\",\n    )\n\n    async with websockets.connect(uri) as ws:\n        # On each binary message: parse the 24-byte header, read frame_id and mask JPEG bytes,\n        # pair with the stored source for that id, decode and composite.\n        # Reference: https://github.com/bria-ai/streaming-rmbg-example\n        pass\n\nasyncio.run(main())\n","lang":"python"},"children":[]}]},"headings":[{"value":"Realtime Video Background Removal API","id":"realtime-video-background-removal-api","depth":1},{"value":"Performance","id":"performance","depth":2},{"value":"Connection","id":"connection","depth":2},{"value":"Authentication","id":"authentication","depth":2},{"value":"API token","id":"api-token","depth":3},{"value":"OAuth","id":"oauth","depth":3},{"value":"Message format","id":"message-format","depth":2},{"value":"Binary frame header (24 bytes)","id":"binary-frame-header-24-bytes","depth":3},{"value":"Constants","id":"constants","depth":3},{"value":"Client → Server","id":"client--server","depth":2},{"value":"Video frame (binary)","id":"video-frame-binary","depth":3},{"value":"Stop session (JSON)","id":"stop-session-json","depth":3},{"value":"Debug mode (JSON)","id":"debug-mode-json","depth":3},{"value":"Server → Client","id":"server--client","depth":2},{"value":"Mask frame (binary)","id":"mask-frame-binary","depth":3},{"value":"Error (JSON)","id":"error-json","depth":3},{"value":"Frame timing (JSON, debug only)","id":"frame-timing-json-debug-only","depth":3},{"value":"WebSocket close codes","id":"websocket-close-codes","depth":2},{"value":"Client-side compositing & optimization","id":"client-side-compositing--optimization","depth":2},{"value":"Algorithm patterns","id":"algorithm-patterns","depth":3},{"value":"Practical tuning","id":"practical-tuning","depth":3},{"value":"Examples","id":"examples","depth":2},{"value":"JavaScript: connect and send","id":"javascript-connect-and-send","depth":3},{"value":"JavaScript: parse mask header","id":"javascript-parse-mask-header","depth":3},{"value":"Python: connect and send","id":"python-connect-and-send","depth":3}],"frontmatter":{"seo":{"title":"Realtime Video Background Removal API"}},"lastModified":"2026-05-04T14:32:43.000Z","pagePropGetterError":{"message":"","name":""}},"slug":"/streaming-rmbg","userData":{"isAuthenticated":false,"teams":["anonymous"]},"isPublic":true}