Files
UNITRONIX c9f7ae5006 fix(agent): hide console windows for PowerShell commands and improve GDI screenshot capture
- Updated PowerShell command invocations to use `-WindowStyle Hidden` to prevent visible console windows during clipboard operations and other commands.
- Implemented GDI-based screenshot capture to eliminate the need for PowerShell, reducing console window spamming during desktop capture.
- Enhanced error handling for missing OpenGL DLLs and ensured complete Mesa DLL set is shipped with the agent to prevent startup issues.
- Updated build scripts to ensure proper installation of required tools and dependencies for Windows builds.
2026-08-09 20:44:12 +02:00

647 lines
20 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package agent
import (
"context"
"os/exec"
"runtime"
"strings"
"sync"
"time"
)
// ── Video codec engine ─────────────────────────────────────────────────────
//
// The agent can encode the captured screen with several codecs. Two families
// exist:
//
// • Image codecs (mjpeg, webp) — each frame is a self-contained still that
// the operator decodes with a plain <img> / createImageBitmap. These work
// with the existing console decoder with NO changes and are the safe
// default. WebP is ~30-50% smaller than MJPEG for the same quality.
//
// • Video codecs (h264, vp9, av1) — a real inter-frame compressed stream.
// These need a WebCodecs VideoDecoder on the operator side. They are far
// more bandwidth efficient and, when a GPU encoder is available, almost
// free on CPU. They are opt-in / negotiated, never forced.
//
// Codec selection precedence (highest first):
// 1. The codec the operator explicitly negotiated via codec_offer, if the
// agent can actually produce it.
// 2. The codec pinned in the agent config (VideoCodec != "" && != "auto").
// 3. Automatic selection based on a one-time probe of ffmpeg's encoders,
// preferring a hardware encoder and the most efficient codec that the
// operator can decode.
//
// Everything degrades gracefully: if a chosen encoder fails to start the
// streamer falls back to the next strategy and ultimately to MJPEG, which is
// guaranteed to work everywhere ffmpeg is installed.
// Codec identifiers used on the wire (desktop_meta.format) and in config.
const (
CodecNone = "none"
CodecMJPEG = "mjpeg"
CodecWebP = "webp"
CodecH264 = "h264"
CodecVP9 = "vp9"
CodecAV1 = "av1"
CodecAuto = "auto"
)
// Hardware acceleration back-ends.
const (
HwAuto = "auto"
HwNone = "none"
HwVAAPI = "vaapi" // Intel/AMD on Linux
HwNVENC = "nvenc" // NVIDIA, all OS
HwQSV = "qsv" // Intel QuickSync
HwAMF = "amf" // AMD on Windows
HwVideoToolbox = "videotoolbox" // Apple
)
// frameMode describes how the encoded output is delimited on the wire.
type frameMode int
const (
// frameModeImage: image2pipe output, one self-contained still per frame
// (JPEG SOI/EOI or WebP RIFF chunk). Decoded by the operator's <img> path.
frameModeImage frameMode = iota
// frameModeAnnexB: H.264/HEVC Annex-B elementary stream. Access units are
// split on the operator-agreed boundary and tagged key/delta by scanning
// NAL unit types (type 5 = IDR).
frameModeAnnexB
// frameModeIVF: VP9/AV1 in an IVF container. Each frame has a 12-byte
// header (4-byte LE size + 8-byte PTS) we strip before forwarding.
frameModeIVF
)
// encoderPlan is the resolved encoder for a session: the concrete ffmpeg
// arguments, the wire codec name and the framing mode.
type encoderPlan struct {
codec string // wire codec: mjpeg|webp|h264|vp9|av1
ffmpegName string // concrete ffmpeg -c:v value (e.g. h264_vaapi)
hwAccel string // resolved hw back-end or "none"
mode frameMode // how output is framed
codecString string // WebCodecs codec string for video codecs
// preInput holds flags that must appear BEFORE -i (hw device init).
preInput []string
}
// ── Encoder capability probe ────────────────────────────────────────────────
// encoderCandidates maps each video codec to the ordered list of ffmpeg
// encoder names to try, hardware first. The first candidate that ffmpeg both
// lists and (for hardware) survives a 1-frame validation encode is used.
var encoderCandidates = map[string][]string{
CodecH264: {"h264_nvenc", "h264_qsv", "h264_vaapi", "h264_amf", "h264_videotoolbox", "libx264"},
CodecVP9: {"vp9_vaapi", "vp9_qsv", "libvpx-vp9"},
CodecAV1: {"av1_nvenc", "av1_qsv", "av1_vaapi", "av1_amf", "libsvtav1", "libaom-av1"},
CodecWebP: {"libwebp"},
CodecMJPEG: {"mjpeg"},
}
// hwOfEncoder maps a concrete ffmpeg encoder to its hardware back-end.
func hwOfEncoder(name string) string {
switch {
case strings.HasSuffix(name, "_nvenc"):
return HwNVENC
case strings.HasSuffix(name, "_qsv"):
return HwQSV
case strings.HasSuffix(name, "_vaapi"):
return HwVAAPI
case strings.HasSuffix(name, "_amf"):
return HwAMF
case strings.HasSuffix(name, "_videotoolbox"):
return HwVideoToolbox
default:
return HwNone
}
}
// codecWebCodecsString returns the WebCodecs codec string the operator must
// pass to VideoDecoder.configure for the given codec.
func codecWebCodecsString(codec string) string {
switch codec {
case CodecH264:
// Baseline-ish; the decoder ignores level for Annex-B with in-band SPS.
return "avc1.42E01F"
case CodecVP9:
return "vp09.00.10.08"
case CodecAV1:
return "av01.0.04M.08"
default:
return ""
}
}
// codecOrder is the efficiency preference used by automatic selection. AV1 is
// the most efficient but the slowest in software; VP9 next; H.264 is the most
// broadly hardware-accelerated and lowest latency. WebP is the safe image
// fallback that needs no operator decoder changes.
var codecOrder = []string{CodecAV1, CodecVP9, CodecH264, CodecWebP}
// encoderProbe caches which ffmpeg encoders are available and validated.
type encoderProbe struct {
once sync.Once
available map[string]bool // ffmpeg lists this encoder
working sync.Map // encoder name -> bool (validated, cached)
ffmpeg string
}
var globalProbe = &encoderProbe{available: map[string]bool{}}
// load lists ffmpeg encoders exactly once per process.
func (p *encoderProbe) load() {
p.once.Do(func() {
ffmpeg, err := exec.LookPath("ffmpeg")
if err != nil {
return
}
p.ffmpeg = ffmpeg
ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second)
defer cancel()
encCmd := exec.CommandContext(ctx, ffmpeg, "-hide_banner", "-encoders")
hideConsole(encCmd)
out, err := encCmd.Output()
if err != nil {
return
}
for _, line := range strings.Split(string(out), "\n") {
f := strings.Fields(strings.TrimSpace(line))
// Lines look like: " V..... h264_vaapi VAAPI H.264 encoder"
if len(f) >= 2 && strings.HasPrefix(f[0], "V") {
p.available[f[1]] = true
}
}
})
}
// listed reports whether ffmpeg advertises the encoder at all.
func (p *encoderProbe) listed(name string) bool {
p.load()
return p.available[name]
}
// validate runs a 1-frame null encode to confirm a hardware encoder actually
// initialises on this machine (drivers/permissions can make a listed encoder
// fail at runtime). Software encoders are trusted without a test. Results are
// cached for the process lifetime.
func (p *encoderProbe) validate(name string) bool {
p.load()
if p.ffmpeg == "" || !p.available[name] {
return false
}
if hwOfEncoder(name) == HwNone {
return true // software encoder — trust it, skip the cost.
}
if v, ok := p.working.Load(name); ok {
return v.(bool)
}
ok := p.testEncode(name)
p.working.Store(name, ok)
return ok
}
// testEncode encodes a single 64x64 synthetic frame to /dev/null and reports
// whether the encoder initialised successfully.
func (p *encoderProbe) testEncode(name string) bool {
ctx, cancel := context.WithTimeout(context.Background(), 6*time.Second)
defer cancel()
args := []string{"-hide_banner", "-loglevel", "error"}
plan := encoderPlan{ffmpegName: name, hwAccel: hwOfEncoder(name)}
args = append(args, hwTestPreInput(plan)...)
args = append(args,
"-f", "lavfi", "-i", "color=c=black:s=64x64:r=5",
"-frames:v", "1",
)
args = append(args, hwTestFilter(plan)...)
args = append(args, "-c:v", name, "-f", "null", "-")
cmd := exec.CommandContext(ctx, p.ffmpeg, args...)
hideConsole(cmd)
return cmd.Run() == nil
}
// hwTestPreInput returns the hw-device init flags needed before -i for the
// validation encode. For lavfi sources the upload filter handles VAAPI/QSV.
func hwTestPreInput(plan encoderPlan) []string {
switch plan.hwAccel {
case HwVAAPI:
return []string{"-init_hw_device", "vaapi=va:" + vaapiDevice(), "-filter_hw_device", "va"}
case HwQSV:
return []string{"-init_hw_device", "qsv=qsv", "-filter_hw_device", "qsv"}
default:
return nil
}
}
// hwTestFilter returns the upload filter chain for the validation encode.
func hwTestFilter(plan encoderPlan) []string {
switch plan.hwAccel {
case HwVAAPI:
return []string{"-vf", "format=nv12,hwupload"}
case HwQSV:
return []string{"-vf", "format=nv12,hwupload=extra_hw_frames=4"}
default:
return nil
}
}
// vaapiDevice returns the VAAPI render node, overridable via env.
func vaapiDevice() string {
// Most systems expose the first render node here.
return "/dev/dri/renderD128"
}
// ── Capability advertisement ────────────────────────────────────────────────
// videoCapabilities returns the ordered list of wire codecs this agent can
// actually produce right now, given the config and a live encoder probe. The
// result feeds codec_offer so the operator only negotiates codecs we can
// deliver. Image codecs are always offered (they need ffmpeg only); video
// codecs are offered when a working encoder exists.
func (a *Agent) videoCapabilities() []string {
if !a.cfg.Screenshot {
return nil
}
caps := []string{}
// Video codecs first (preferred), validated.
for _, codec := range []string{CodecAV1, CodecVP9, CodecH264} {
if a.codecAllowed(codec) && resolveEncoder(codec, a.cfg.HwAccel) != nil {
caps = append(caps, codec)
}
}
// Image codecs are always available when ffmpeg is present; webp needs
// libwebp. MJPEG is the universal guarantee and is always last.
if globalProbe.validate("libwebp") {
caps = append(caps, CodecWebP)
}
caps = append(caps, CodecMJPEG)
return caps
}
// codecAllowed honours an explicit per-codec pin. When VideoCodec is a concrete
// codec, only that codec (and the image fallbacks) are advertised.
func (a *Agent) codecAllowed(codec string) bool {
pin := strings.ToLower(strings.TrimSpace(a.cfg.VideoCodec))
if pin == "" || pin == CodecAuto {
return true
}
return pin == codec
}
// resolveEncoder returns a usable encoderPlan for the requested codec under the
// given hw preference, or nil if no working encoder exists. hwPref of "" or
// "auto" lets the probe pick the best; a concrete back-end restricts to it.
func resolveEncoder(codec, hwPref string) *encoderPlan {
hwPref = strings.ToLower(strings.TrimSpace(hwPref))
for _, name := range encoderCandidates[codec] {
hw := hwOfEncoder(name)
switch hwPref {
case "", HwAuto:
// any
case HwNone:
if hw != HwNone {
continue
}
default:
if hw != hwPref && hw != HwNone {
continue
}
// Allow software fallback only when no hw candidate matched; the
// loop order already prefers hardware, so a software encoder here
// means the user asked for a back-end this codec can't use — still
// give them the codec via software rather than nothing.
}
if !globalProbe.validate(name) {
continue
}
return buildPlan(codec, name)
}
return nil
}
// buildPlan fills in the framing mode and WebCodecs string for a resolved
// encoder name.
func buildPlan(codec, ffmpegName string) *encoderPlan {
mode := frameModeImage
switch codec {
case CodecH264:
mode = frameModeAnnexB
case CodecVP9, CodecAV1:
mode = frameModeIVF
}
return &encoderPlan{
codec: codec,
ffmpegName: ffmpegName,
hwAccel: hwOfEncoder(ffmpegName),
mode: mode,
codecString: codecWebCodecsString(codec),
}
}
// selectEncoder resolves the encoder to use for a session. requested is the
// codec the operator pinned (may be "" or "auto"). opCodecs is the list of
// codecs the operator can actually DECODE (from the desktop_start payload).
// It honours, in order: the operator's pinned+decodable request, the config
// pin, then automatic best-codec selection limited to what the operator can
// decode. It always succeeds because MJPEG is the universal last resort.
//
// Backward compatibility: a legacy operator that sends no codec list can only
// decode JPEG, so the result is forced to MJPEG regardless of agent ability.
func (a *Agent) selectEncoder(requested string, opCodecs []string) encoderPlan {
hw := a.cfg.HwAccel
want := normalizeWireCodec(requested)
pin := normalizeWireCodec(a.cfg.VideoCodec)
// Normalize and index the operator's decode capabilities. "jpeg" is the
// historical wire name for MJPEG.
canDecode := map[string]bool{}
for _, c := range opCodecs {
canDecode[normalizeWireCodec(c)] = true
}
legacyOperator := len(canDecode) == 0
opDecodes := func(codec string) bool {
if codec == CodecMJPEG {
return true // every operator decodes JPEG
}
return canDecode[codec]
}
produce := func(codec string) (encoderPlan, bool) {
if !a.codecAllowed(codec) || !opDecodes(codec) {
return encoderPlan{}, false
}
switch codec {
case CodecMJPEG:
return imagePlan(CodecMJPEG), true
case CodecWebP:
if globalProbe.validate("libwebp") {
return imagePlan(CodecWebP), true
}
default:
if p := resolveEncoder(codec, hw); p != nil {
return *p, true
}
}
return encoderPlan{}, false
}
if legacyOperator {
return imagePlan(CodecMJPEG)
}
// 1. Operator's explicit, decodable request.
if want != "" && want != CodecAuto {
if p, ok := produce(want); ok {
return p
}
}
// 2. Config pin (concrete codec).
if pin != "" && pin != CodecAuto {
if p, ok := produce(pin); ok {
return p
}
}
// 3. Automatic: most efficient video codec the operator can decode and the
// agent can produce (hardware preferred via candidate ordering).
for _, codec := range codecOrder {
if p, ok := produce(codec); ok {
return p
}
}
// 4. Universal last resort.
return imagePlan(CodecMJPEG)
}
// normalizeWireCodec lowercases and maps the historical "jpeg" alias to
// CodecMJPEG.
func normalizeWireCodec(c string) string {
c = strings.ToLower(strings.TrimSpace(c))
if c == "jpeg" {
return CodecMJPEG
}
return c
}
// imagePlan builds an image-codec plan (mjpeg/webp).
func imagePlan(codec string) encoderPlan {
name := "mjpeg"
if codec == CodecWebP {
name = "libwebp"
}
return encoderPlan{codec: codec, ffmpegName: name, hwAccel: HwNone, mode: frameModeImage}
}
// ── Encoder argument construction ───────────────────────────────────────────
// encoderTail returns the ffmpeg arguments that follow the capture input for
// this plan: optional hw upload filter, the codec, codec options and the
// output muxer. quality is 0-100, fps is the target frame rate.
//
// preInput returns flags that must precede -i (hardware device init); the
// caller is responsible for inserting them before the input args.
func (plan encoderPlan) preInputArgs() []string {
switch plan.hwAccel {
case HwVAAPI:
return []string{"-init_hw_device", "vaapi=va:" + vaapiDevice(), "-filter_hw_device", "va"}
case HwQSV:
return []string{"-init_hw_device", "qsv=qsv", "-filter_hw_device", "qsv"}
default:
return nil
}
}
func (plan encoderPlan) encoderTail(fps, quality int) []string {
if quality < 1 {
quality = 60
}
if quality > 100 {
quality = 100
}
if fps < 1 {
fps = 15
}
switch plan.mode {
case frameModeImage:
return plan.imageTail(fps, quality)
case frameModeAnnexB:
return plan.h264Tail(fps, quality)
case frameModeIVF:
return plan.ivfTail(fps, quality)
}
return plan.imageTail(fps, quality)
}
// imageTail builds the mjpeg/webp image2pipe encoder.
func (plan encoderPlan) imageTail(fps, quality int) []string {
if plan.codec == CodecWebP {
// libwebp quality is 0-100 directly. -lossless 0 keeps it lossy/small.
return []string{
"-vf", "fps=" + itoa(fps),
"-c:v", "libwebp",
"-lossless", "0",
"-quality", itoa(quality),
"-preset", "picture",
"-f", "image2pipe",
"-",
}
}
// MJPEG: ffmpeg q:v scale is 2 (best) 31 (worst).
mq := 31 - (quality * 29 / 100)
if mq < 2 {
mq = 2
}
return []string{
"-vf", "fps=" + itoa(fps),
"-vcodec", "mjpeg",
"-q:v", itoa(mq),
"-f", "image2pipe",
"-",
}
}
// h264Tail builds the H.264 Annex-B encoder. A short GOP and in-band SPS/PPS
// (Annex-B) let the WebCodecs decoder start without an out-of-band config.
func (plan encoderPlan) h264Tail(fps, quality int) []string {
gop := itoa(fps * 2)
out := []string{}
switch plan.hwAccel {
case HwVAAPI:
out = append(out, "-vf", "format=nv12,hwupload")
case HwQSV:
out = append(out, "-vf", "format=nv12,hwupload=extra_hw_frames=8")
}
out = append(out, "-c:v", plan.ffmpegName)
switch plan.hwAccel {
case HwNVENC:
out = append(out, "-preset", "p4", "-tune", "ll", "-rc", "vbr", "-cq", itoa(qToCQ(quality)))
case HwVAAPI:
out = append(out, "-rc_mode", "CQP", "-qp", itoa(qToQP(quality)))
case HwQSV:
out = append(out, "-global_quality", itoa(qToQP(quality)))
case HwAMF:
out = append(out, "-quality", "speed", "-rc", "cqp", "-qp_i", itoa(qToQP(quality)), "-qp_p", itoa(qToQP(quality)))
case HwVideoToolbox:
out = append(out, "-q:v", itoa(quality))
default: // libx264
out = append(out, "-preset", "veryfast", "-tune", "zerolatency", "-crf", itoa(qToCRF(quality)))
}
out = append(out,
"-g", gop,
"-bf", "0", // no B-frames: lowest latency, simplest AU boundaries
"-pix_fmt", "yuv420p",
"-bsf:v", "h264_mp4toannexb", // harmless if already annexb
"-f", "h264",
"-",
)
return out
}
// ivfTail builds the VP9/AV1 encoder muxed into IVF for easy frame splitting.
func (plan encoderPlan) ivfTail(fps, quality int) []string {
gop := itoa(fps * 2)
out := []string{}
switch plan.hwAccel {
case HwVAAPI:
out = append(out, "-vf", "format=nv12,hwupload")
case HwQSV:
out = append(out, "-vf", "format=nv12,hwupload=extra_hw_frames=8")
}
out = append(out, "-c:v", plan.ffmpegName)
switch plan.codec {
case CodecVP9:
switch plan.hwAccel {
case HwVAAPI:
out = append(out, "-rc_mode", "CQP", "-qp", itoa(qToQP(quality)))
case HwQSV:
out = append(out, "-global_quality", itoa(qToQP(quality)))
default: // libvpx-vp9
out = append(out, "-deadline", "realtime", "-cpu-used", "8", "-crf", itoa(qToCRF(quality)), "-b:v", "0")
}
case CodecAV1:
switch plan.hwAccel {
case HwNVENC:
out = append(out, "-preset", "p4", "-rc", "vbr", "-cq", itoa(qToCQ(quality)))
case HwQSV:
out = append(out, "-global_quality", itoa(qToQP(quality)))
case HwVAAPI:
out = append(out, "-rc_mode", "CQP", "-qp", itoa(qToQP(quality)))
case HwAMF:
out = append(out, "-rc", "cqp", "-qp_i", itoa(qToQP(quality)), "-qp_p", itoa(qToQP(quality)))
default:
// libsvtav1 (preferred sw) vs libaom-av1.
if plan.ffmpegName == "libsvtav1" {
out = append(out, "-preset", "10", "-crf", itoa(qToCRF(quality)))
} else {
out = append(out, "-usage", "realtime", "-cpu-used", "8", "-crf", itoa(qToCRF(quality)), "-b:v", "0")
}
}
}
out = append(out,
"-g", gop,
"-pix_fmt", "yuv420p",
"-f", "ivf",
"-",
)
return out
}
// qToCRF / qToQP / qToCQ map a 0-100 quality slider to codec quantiser values.
// Higher slider = better quality = lower quantiser.
func qToCRF(q int) int { return clampQuant(63 - (q * 53 / 100)) } // 10 (best) 63
func qToQP(q int) int { return clampQuant(51 - (q * 41 / 100)) } // 10 (best) 51
func qToCQ(q int) int { return clampQuant(51 - (q * 41 / 100)) }
func clampQuant(v int) int {
if v < 1 {
return 1
}
if v > 63 {
return 63
}
return v
}
// itoa is a tiny strconv-free int formatter for the hot encoder-arg path.
func itoa(v int) string {
if v == 0 {
return "0"
}
neg := v < 0
if neg {
v = -v
}
var b [20]byte
i := len(b)
for v > 0 {
i--
b[i] = byte('0' + v%10)
v /= 10
}
if neg {
i--
b[i] = '-'
}
return string(b[i:])
}
// platformDefaultHwAccel returns the most likely hardware back-end for the OS,
// used only for logging/diagnostics; actual selection is probe-driven.
func platformDefaultHwAccel() string {
switch runtime.GOOS {
case "windows":
return HwNVENC
case "darwin":
return HwVideoToolbox
default:
return HwVAAPI
}
}