diff --git a/patches/004-h264-passthrough.patch b/patches/004-h264-passthrough.patch new file mode 100644 index 0000000..707b1ef --- /dev/null +++ b/patches/004-h264-passthrough.patch @@ -0,0 +1,172 @@ +diff --git a/src/protocols/rdp/channels/rdpgfx.c b/src/protocols/rdp/channels/rdpgfx.c +index 327e7c2b..260681f2 100644 +--- a/src/protocols/rdp/channels/rdpgfx.c ++++ b/src/protocols/rdp/channels/rdpgfx.c +@@ -24,14 +24,120 @@ + #include "settings.h" + + #include ++#include + #include + #include + #include + #include ++#include ++#include ++#include + ++#include + #include + #include + ++/** ++ * The original SurfaceCommand handler installed by gdi_graphics_pipeline_init(). ++ */ ++static pcRdpgfxSurfaceCommand guac_rdpgfx_orig_surface_command = NULL; ++ ++/** ++ * Check if a H.264 NAL unit buffer contains a keyframe (IDR slice). ++ */ ++static int guac_rdp_h264_is_keyframe(const BYTE* data, UINT32 length) { ++ for (UINT32 i = 0; i + 3 < length; i++) { ++ if (data[i] == 0 && data[i+1] == 0) { ++ int offset = -1; ++ if (data[i+2] == 1) ++ offset = i + 3; ++ else if (i + 4 < length && data[i+2] == 0 && data[i+3] == 1) ++ offset = i + 4; ++ if (offset >= 0 && (UINT32) offset < length) { ++ int nal_type = data[offset] & 0x1F; ++ if (nal_type == 5 || nal_type == 7) ++ return 1; ++ } ++ } ++ } ++ return 0; ++} ++ ++/** ++ * Send raw H.264 NAL units to the Guacamole client as a custom "h264" ++ * instruction followed by blob data and an end instruction. ++ */ ++static void guac_rdp_h264_send_passthrough(guac_client* client, ++ const RDPGFX_SURFACE_COMMAND* cmd, ++ const BYTE* h264_data, UINT32 h264_length) { ++ ++ guac_socket* socket = client->socket; ++ guac_stream* stream = guac_client_alloc_stream(client); ++ int keyframe = guac_rdp_h264_is_keyframe(h264_data, h264_length); ++ ++ /* Build and send h264 instruction header */ ++ char arg_bufs[7][16]; ++ int vals[] = { stream->index, 0, keyframe, ++ (int) cmd->left, (int) cmd->top, ++ (int) cmd->width, (int) cmd->height }; ++ ++ guac_socket_instruction_begin(socket); ++ guac_socket_write_string(socket, "4.h264"); ++ for (int i = 0; i < 7; i++) { ++ int n = snprintf(arg_bufs[i], sizeof(arg_bufs[i]), "%d", vals[i]); ++ char len_buf[8]; ++ snprintf(len_buf, sizeof(len_buf), ",%d.", n); ++ guac_socket_write_string(socket, len_buf); ++ guac_socket_write_string(socket, arg_bufs[i]); ++ } ++ guac_socket_write_string(socket, ";"); ++ guac_socket_instruction_end(socket); ++ ++ guac_protocol_send_blobs(socket, stream, h264_data, h264_length); ++ guac_protocol_send_end(socket, stream); ++ guac_client_free_stream(client, stream); ++ ++ guac_client_log(client, GUAC_LOG_TRACE, ++ "H.264 passthrough: %u bytes, %s, %ux%u at (%u,%u)", ++ h264_length, keyframe ? "keyframe" : "delta", ++ cmd->width, cmd->height, cmd->left, cmd->top); ++} ++ ++/** ++ * Wrapper for the FreeRDP GDI SurfaceCommand callback. Intercepts AVC420 ++ * and AVC444 commands to extract raw H.264 NAL units for passthrough. ++ */ ++static UINT guac_rdp_gfx_surface_command(RdpgfxClientContext* context, ++ const RDPGFX_SURFACE_COMMAND* cmd) { ++ ++ rdpContext* rdp_context = (rdpContext*) context->custom; ++ guac_client* client = ((rdp_freerdp_context*) rdp_context)->client; ++ ++ if (cmd->codecId == RDPGFX_CODECID_AVC420 && cmd->extra != NULL) { ++ RDPGFX_AVC420_BITMAP_STREAM* avc420 = ++ (RDPGFX_AVC420_BITMAP_STREAM*) cmd->extra; ++ if (avc420->data != NULL && avc420->length > 0) ++ guac_rdp_h264_send_passthrough(client, cmd, ++ avc420->data, avc420->length); ++ } ++ else if ((cmd->codecId == RDPGFX_CODECID_AVC444 ++ || cmd->codecId == RDPGFX_CODECID_AVC444v2) ++ && cmd->extra != NULL) { ++ RDPGFX_AVC444_BITMAP_STREAM* avc444 = ++ (RDPGFX_AVC444_BITMAP_STREAM*) cmd->extra; ++ if (avc444->bitstream[0].data != NULL ++ && avc444->bitstream[0].length > 0) ++ guac_rdp_h264_send_passthrough(client, cmd, ++ avc444->bitstream[0].data, ++ avc444->bitstream[0].length); ++ } ++ ++ if (guac_rdpgfx_orig_surface_command != NULL) ++ return guac_rdpgfx_orig_surface_command(context, cmd); ++ ++ return CHANNEL_RC_OK; ++} ++ + /** + * Callback which associates handlers specific to Guacamole with the + * RdpgfxClientContext instance allocated by FreeRDP to deal with received +@@ -63,13 +169,22 @@ static void guac_rdp_rdpgfx_channel_connected(rdpContext* context, + RdpgfxClientContext* rdpgfx = (RdpgfxClientContext*) args->pInterface; + rdpGdi* gdi = context->gdi; + +- if (!gdi_graphics_pipeline_init(gdi, rdpgfx)) ++ if (!gdi_graphics_pipeline_init(gdi, rdpgfx)) { + guac_client_log(client, GUAC_LOG_WARNING, "Rendering backend for RDPGFX " + "channel could not be loaded. Graphics may not render at all!"); +- else ++ } ++ else { + guac_client_log(client, GUAC_LOG_DEBUG, "RDPGFX channel will be used for " + "the RDP Graphics Pipeline Extension."); + ++ /* Wrap SurfaceCommand to intercept H.264 data before GDI decode */ ++ guac_rdpgfx_orig_surface_command = rdpgfx->SurfaceCommand; ++ rdpgfx->SurfaceCommand = guac_rdp_gfx_surface_command; ++ ++ guac_client_log(client, GUAC_LOG_INFO, ++ "H.264 passthrough enabled for RDPGFX channel."); ++ } ++ + } + + /** +diff --git a/src/protocols/rdp/settings.c b/src/protocols/rdp/settings.c +index d97d98e1..bd11f6da 100644 +--- a/src/protocols/rdp/settings.c ++++ b/src/protocols/rdp/settings.c +@@ -1619,6 +1619,8 @@ void guac_rdp_push_settings(guac_client* client, + + freerdp_settings_set_bool(rdp_settings, FreeRDP_SupportGraphicsPipeline, TRUE); + freerdp_settings_set_bool(rdp_settings, FreeRDP_RemoteFxCodec, TRUE); ++ freerdp_settings_set_bool(rdp_settings, FreeRDP_GfxH264, TRUE); ++ freerdp_settings_set_bool(rdp_settings, FreeRDP_GfxAVC444, TRUE); + + if (freerdp_settings_get_uint32(rdp_settings, FreeRDP_ColorDepth) != RDP_GFX_REQUIRED_DEPTH) { + guac_client_log(client, GUAC_LOG_WARNING, "Ignoring requested " +@@ -1878,6 +1880,8 @@ void guac_rdp_push_settings(guac_client* client, + + rdp_settings->SupportGraphicsPipeline = TRUE; + rdp_settings->RemoteFxCodec = TRUE; ++ rdp_settings->GfxH264 = TRUE; ++ rdp_settings->GfxAVC444 = TRUE; + + if (rdp_settings->ColorDepth != RDP_GFX_REQUIRED_DEPTH) { + guac_client_log(client, GUAC_LOG_WARNING, "Ignoring requested " diff --git a/src/guacd.rs b/src/guacd.rs index c6191ef..f808d03 100644 --- a/src/guacd.rs +++ b/src/guacd.rs @@ -439,7 +439,7 @@ async fn send_handshake( vec![width.to_string(), height.to_string(), dpi.to_string()], ), Instruction::new("audio", vec!["audio/L16".into(), "audio/L8".into()]), - Instruction::new("video", vec![]), + Instruction::new("video", vec!["video/h264".into()]), Instruction::new( "image", vec!["image/png".into(), "image/jpeg".into(), "image/webp".into()], diff --git a/static/client.html b/static/client.html index c3def3b..f99658c 100644 --- a/static/client.html +++ b/static/client.html @@ -101,6 +101,7 @@ + diff --git a/static/guac/Client.js b/static/guac/Client.js index 14dab6f..fc67ddd 100644 --- a/static/guac/Client.js +++ b/static/guac/Client.js @@ -1431,6 +1431,63 @@ Guacamole.Client = function(tunnel) { }, + "h264": function(parameters) { + + var stream_index = parseInt(parameters[0]); + var layer = getLayer(parseInt(parameters[1])); + var isKeyFrame = parseInt(parameters[2]) !== 0; + var x = parseInt(parameters[3]); + var y = parseInt(parameters[4]); + var width = parseInt(parameters[5]); + var height = parseInt(parameters[6]); + + // Create stream to receive H.264 NAL unit data + var stream = streams[stream_index] = new Guacamole.InputStream(guac_client, stream_index); + + // Check for WebCodecs support + if (!Guacamole.H264Decoder.isSupported()) { + // No WebCodecs: acknowledge stream but discard data + stream.onblob = function() {}; + stream.onend = function() {}; + guac_client.sendAck(stream_index, "OK", 0x0000); + return; + } + + // Create or reuse H.264 decoder for this display + if (!guac_client._h264Decoder) { + guac_client._h264Decoder = new Guacamole.H264Decoder(display); + } + + // Collect base64 blob data + var base64Data = ''; + stream.onblob = function(data) { + base64Data += data; + }; + + stream.onend = function() { + if (!base64Data) return; + + // Decode base64 to ArrayBuffer + try { + var binaryString = atob(base64Data); + var bytes = new Uint8Array(binaryString.length); + for (var i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + + // Feed to H.264 decoder + guac_client._h264Decoder.decode( + layer, x, y, width, height, + bytes.buffer, isKeyFrame + ); + } catch (e) { + if (typeof console !== 'undefined') + console.error('[rustguac] H.264 base64 decode error:', e); + } + }; + + }, + "jpeg": function(parameters) { var channelMask = parseInt(parameters[0]); diff --git a/static/guac/H264Decoder.js b/static/guac/H264Decoder.js new file mode 100644 index 0000000..fe6aab8 --- /dev/null +++ b/static/guac/H264Decoder.js @@ -0,0 +1,210 @@ +/* + * H.264 decoder for Guacamole using the WebCodecs API. + * Decodes H.264 NAL units received via the "h264" instruction and + * renders decoded frames to a Guacamole Display layer. + * + * Copyright (C) 2026 Sol1 Pty Ltd. Licensed under Apache 2.0. + */ + +var Guacamole = Guacamole || {}; + +/** + * H.264 video decoder that uses the WebCodecs VideoDecoder API for + * hardware-accelerated decoding of H.264 NAL units received from guacd. + * + * @constructor + * @param {!Guacamole.Display} display + * The Guacamole display to render decoded frames to. + */ +Guacamole.H264Decoder = function H264Decoder(display) { + + /** + * The WebCodecs VideoDecoder instance, or null if not yet initialised + * or if WebCodecs is not supported. + * + * @private + * @type {?VideoDecoder} + */ + var decoder = null; + + /** + * Whether the decoder has been configured with codec parameters. + * + * @private + * @type {boolean} + */ + var configured = false; + + /** + * Monotonic timestamp counter for EncodedVideoChunk (microseconds). + * + * @private + * @type {number} + */ + var timestamp = 0; + + /** + * The target layer for rendering decoded frames. + * + * @private + * @type {?Guacamole.Display.VisibleLayer} + */ + var targetLayer = null; + + /** + * Pending draw position for the current frame. + * + * @private + */ + var pendingX = 0; + var pendingY = 0; + + /** + * Total frames decoded. + * + * @type {number} + */ + this.framesDecoded = 0; + + /** + * Total frames dropped or errored. + * + * @type {number} + */ + this.framesDropped = 0; + + /** + * Reference to this for closures. + */ + var self = this; + + /** + * Initialise the VideoDecoder if not already done. + * + * @private + * @param {number} width - Expected frame width. + * @param {number} height - Expected frame height. + */ + function ensureDecoder(width, height) { + + if (decoder && configured) + return; + + if (typeof VideoDecoder === 'undefined') { + console.warn('[rustguac] WebCodecs VideoDecoder not available'); + return; + } + + decoder = new VideoDecoder({ + output: function(frame) { + self.framesDecoded++; + try { + // Draw the decoded VideoFrame directly to the layer's canvas + if (targetLayer) { + var canvas = targetLayer.getCanvas(); + var ctx = canvas.getContext('2d'); + ctx.drawImage(frame, pendingX, pendingY); + } + } finally { + // CRITICAL: always close VideoFrame to release GPU memory + frame.close(); + } + }, + error: function(e) { + self.framesDropped++; + console.error('[rustguac] H.264 decode error:', e.message); + } + }); + + // Configure for H.264 Constrained Baseline (most compatible) + // The actual profile/level will be negotiated by the RDP server + decoder.configure({ + codec: 'avc1.42001f', // Baseline profile, level 3.1 + optimizeForLatency: true + }); + + configured = true; + console.log('[rustguac] H.264 WebCodecs decoder initialised (' + width + 'x' + height + ')'); + } + + /** + * Decode a complete H.264 NAL unit buffer and render to the given layer. + * + * @param {!Guacamole.Display.VisibleLayer} layer + * The layer to draw the decoded frame to. + * @param {number} x - X position on the layer. + * @param {number} y - Y position on the layer. + * @param {number} width - Frame width. + * @param {number} height - Frame height. + * @param {!ArrayBuffer} nalData - Raw H.264 NAL unit data (Annex B format). + * @param {boolean} isKeyFrame - Whether this contains an IDR/keyframe. + */ + this.decode = function(layer, x, y, width, height, nalData, isKeyFrame) { + + ensureDecoder(width, height); + + if (!decoder || decoder.state === 'closed') + return; + + targetLayer = layer; + pendingX = x; + pendingY = y; + + try { + var chunk = new EncodedVideoChunk({ + type: isKeyFrame ? 'key' : 'delta', + timestamp: timestamp, + data: nalData + }); + timestamp += 33333; // ~30fps in microseconds + + decoder.decode(chunk); + } catch (e) { + self.framesDropped++; + console.error('[rustguac] H.264 chunk error:', e.message); + } + }; + + /** + * Reset the decoder (e.g. after reconnection or error recovery). + * The next frame must be a keyframe. + */ + this.reset = function() { + if (decoder && decoder.state !== 'closed') { + try { + decoder.reset(); + configured = false; + timestamp = 0; + console.log('[rustguac] H.264 decoder reset'); + } catch (e) { + // Decoder may be in error state + } + } + }; + + /** + * Close and release the decoder. + */ + this.destroy = function() { + if (decoder && decoder.state !== 'closed') { + try { + decoder.close(); + } catch (e) { + // Ignore + } + } + decoder = null; + configured = false; + }; + +}; + +/** + * Check if the browser supports H.264 decoding via WebCodecs. + * + * @returns {boolean} + * true if WebCodecs VideoDecoder is available and supports H.264. + */ +Guacamole.H264Decoder.isSupported = function isSupported() { + return typeof VideoDecoder !== 'undefined'; +};