Skip to content

Commit 79a494f

Browse files
purpleempresspurpleempress
committed
Fix Wayland VA-API capture of DMA-BUF frames
1 parent ae06722 commit 79a494f

2 files changed

Lines changed: 92 additions & 26 deletions

File tree

‎internal/airplay/capture.go‎

Lines changed: 59 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -804,6 +804,10 @@ func buildGstVideoPipeline(source gstStage, beforeConvert, afterScale []gstStage
804804
if encoder.needsVulkan {
805805
args = appendGstStage(args, gstStage{"vulkanupload"})
806806
}
807+
return appendGstVideoEncoding(args, encoder, timestampedOutput)
808+
}
809+
810+
func appendGstVideoEncoding(args []string, encoder encoderResult, timestampedOutput bool) []string {
807811
args = appendGstStage(args, encoder.parts)
808812
parser, mediaType, payloader := "h264parse", "video/x-h264", "rtph264pay"
809813
if encoder.codec == VideoCodecHEVC {
@@ -827,6 +831,30 @@ func buildGstVideoPipeline(source gstStage, beforeConvert, afterScale []gstStage
827831
return appendGstStage(args, gstStage{"fdsink", "fd=1", "sync=false", "async=false"})
828832
}
829833

834+
// buildVAWaylandVideoPipeline imports portal buffers without a CPU copy, then
835+
// keeps conversion, scaling, and encoding in VA memory. PipeWire keepalives
836+
// supply idle frames; videorate limits them to the requested output rate.
837+
func buildVAWaylandVideoPipeline(fd int, nodeID uint32, fps int, encoder encoderResult, maxWidth, maxHeight int, timestampedOutput bool) []string {
838+
source := gstStage{"pipewiresrc", fmt.Sprintf("fd=%d", fd), fmt.Sprintf("path=%d", nodeID),
839+
"do-timestamp=true", fmt.Sprintf("keepalive-time=%d", frameIntervalMillis(fps)), "always-copy=false"}
840+
args := append([]string{"--quiet"}, source...)
841+
// Desktop pixels are square. Without this constraint, the VA transform can
842+
// negotiate the minimum of its PAR range and fail to calculate borders.
843+
args = appendGstStage(args, gstStage{"video/x-raw(ANY),pixel-aspect-ratio=1/1"})
844+
// Force a fresh surface even when the source already matches the output.
845+
// Downstream retains the converted surface instead of another portal buffer.
846+
args = appendGstStage(args, gstStage{"vapostproc", "disable-passthrough=true", "add-borders=true"})
847+
caps := "video/x-raw(memory:VAMemory),format=NV12"
848+
if maxWidth > 1 && maxHeight > 1 {
849+
caps += fmt.Sprintf(",width=%d,height=%d,pixel-aspect-ratio=1/1", maxWidth&^1, maxHeight&^1)
850+
}
851+
args = appendGstStage(args, gstStage{caps})
852+
args = appendGstStage(args, gstStage{"videorate", "drop-only=true", "skip-to-first=true"})
853+
args = appendGstStage(args, gstStage{caps + fmt.Sprintf(",framerate=%d/1", fps)})
854+
args = appendGstStage(args, lowLatencyVideoQueueStage())
855+
return appendGstVideoEncoding(args, encoder, timestampedOutput)
856+
}
857+
830858
func startPreparedWaylandCapture(ctx context.Context, cfg CaptureConfig, encoderParts encoderResult, nodeID uint32, pwFd *os.File, dbusConn *dbus.Conn, streamSize [2]int, timestampedOutput bool) (*ScreenCapture, error) {
831859
if pwFd == nil || dbusConn == nil {
832860
if pwFd != nil {
@@ -853,36 +881,41 @@ func startPreparedWaylandCapture(ctx context.Context, cfg CaptureConfig, encoder
853881
// The encoded dimensions are capped to the receiver's advertised display size
854882
// when available. The actual result is read back from the codec SPS downstream.
855883
const pwFdNum = 3
856-
source := pipeWireVideoSourceStage(pwFdNum, nodeID, fps)
884+
var gstArgs []string
885+
if len(encoderParts.parts) > 0 && encoderParts.parts[0] == "vah264enc" && hasGstElement("vapostproc") {
886+
gstArgs = buildVAWaylandVideoPipeline(pwFdNum, nodeID, fps, encoderParts, cfg.MaxWidth, cfg.MaxHeight, timestampedOutput)
887+
} else {
888+
source := pipeWireVideoSourceStage(pwFdNum, nodeID, fps)
857889

858-
hasCompositor := streamSize[0] > 0 && streamSize[1] > 0 && hasGstElement("compositor")
890+
hasCompositor := streamSize[0] > 0 && streamSize[1] > 0 && hasGstElement("compositor")
859891

860-
var beforeConvert []gstStage
861-
if hasGstElement("vapostproc") {
862-
beforeConvert = append(beforeConvert, gstStage{"vapostproc"})
863-
} else {
864-
log.Printf("[CAPTURE] vapostproc unavailable, using software conversion")
865-
}
892+
var beforeConvert []gstStage
893+
if hasGstElement("vapostproc") {
894+
beforeConvert = append(beforeConvert, gstStage{"vapostproc"})
895+
} else {
896+
log.Printf("[CAPTURE] vapostproc unavailable, using software conversion")
897+
}
866898

867-
var afterScale []gstStage
868-
if hasCompositor {
869-
beforeConvert = append(beforeConvert,
870-
gstStage{"compositor", "force-live=true", "ignore-inactive-pads=true", "background=black"},
871-
gstStage{fmt.Sprintf("video/x-raw,width=%d,height=%d,framerate=%d/1", streamSize[0], streamSize[1], fps)},
872-
)
873-
} else {
874-
log.Printf("[CAPTURE] idle-frame compositor unavailable; using portal frame timing")
875-
}
876-
if hasCompositor {
877-
afterScale = append(afterScale, lowLatencyVideoQueueStage())
878-
} else {
879-
afterScale = append(afterScale,
880-
gstStage{"videorate", "drop-only=true", "skip-to-first=true"},
881-
frameRateStage(fps),
882-
lowLatencyVideoQueueStage(),
883-
)
899+
var afterScale []gstStage
900+
if hasCompositor {
901+
beforeConvert = append(beforeConvert,
902+
gstStage{"compositor", "force-live=true", "ignore-inactive-pads=true", "background=black"},
903+
gstStage{fmt.Sprintf("video/x-raw,width=%d,height=%d,framerate=%d/1", streamSize[0], streamSize[1], fps)},
904+
)
905+
} else {
906+
log.Printf("[CAPTURE] idle-frame compositor unavailable; using portal frame timing")
907+
}
908+
if hasCompositor {
909+
afterScale = append(afterScale, lowLatencyVideoQueueStage())
910+
} else {
911+
afterScale = append(afterScale,
912+
gstStage{"videorate", "drop-only=true", "skip-to-first=true"},
913+
frameRateStage(fps),
914+
lowLatencyVideoQueueStage(),
915+
)
916+
}
917+
gstArgs = buildGstVideoPipeline(source, beforeConvert, afterScale, encoderParts, cfg.MaxWidth, cfg.MaxHeight, timestampedOutput)
884918
}
885-
gstArgs := buildGstVideoPipeline(source, beforeConvert, afterScale, encoderParts, cfg.MaxWidth, cfg.MaxHeight, timestampedOutput)
886919

887920
dbg("[CAPTURE] gst-launch-1.0 (wayland) %s", strings.Join(gstArgs, " "))
888921
cmd := exec.CommandContext(captureCtx, "gst-launch-1.0", gstArgs...)

‎internal/airplay/capture_test.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package airplay
33
import (
44
"context"
55
"errors"
6+
"fmt"
67
"os"
78
"os/exec"
89
"reflect"
@@ -810,3 +811,35 @@ func TestDetectGstEncoderSelectionContract(t *testing.T) {
810811
})
811812
}
812813
}
814+
815+
func TestVAWaylandPipelineKeepsFramesInVAMemory(t *testing.T) {
816+
encoder := encoderResult{parts: gstStage{"vah264enc"}, rawFormat: "NV12", codec: VideoCodecH264}
817+
for _, size := range [][2]int{{0, 0}, {1920, 1080}, {1279, 719}, {1, 1}} {
818+
pipeline := buildVAWaylandVideoPipeline(3, 42, 30, encoder, size[0], size[1], true)
819+
joined := strings.Join(pipeline, " ")
820+
for _, forbidden := range []string{"always-copy=true", "videoconvert", "videoscale", "compositor"} {
821+
if strings.Contains(joined, forbidden) {
822+
t.Errorf("VA pipeline must not copy or process portal frames on the CPU: %s", joined)
823+
}
824+
}
825+
for _, required := range []string{"keepalive-time=33", "always-copy=false", "disable-passthrough=true", "add-borders=true", "video/x-raw(ANY),pixel-aspect-ratio=1/1", "video/x-raw(memory:VAMemory),format=NV12"} {
826+
if !strings.Contains(joined, required) {
827+
t.Errorf("VA pipeline is missing %q: %s", required, joined)
828+
}
829+
}
830+
if size[0] > 1 && size[1] > 1 {
831+
want := fmt.Sprintf("width=%d,height=%d,pixel-aspect-ratio=1/1", size[0]&^1, size[1]&^1)
832+
if !strings.Contains(joined, want) {
833+
t.Errorf("VA scaling must use an even receiver canvas: %s", joined)
834+
}
835+
} else if strings.Contains(joined, "width=") || strings.Contains(joined, "height=") {
836+
t.Errorf("invalid receiver size must not constrain the capture: %s", joined)
837+
}
838+
// The VA path must retain the same timestamp-preserving output as other
839+
// sources, since the sender schedules video from the encoded buffer PTS.
840+
suffix := appendGstVideoEncoding(nil, encoder, true)
841+
if !reflect.DeepEqual(pipeline[len(pipeline)-len(suffix):], suffix) {
842+
t.Errorf("VA pipeline changed the shared encoding/output suffix: %s", joined)
843+
}
844+
}
845+
}

0 commit comments

Comments
 (0)