Skip to content

Commit 63f248f

Browse files
committed
fix(linux): skip h264_vulkan when the device has no H.264 encode queue
ffmpeg 8.1 segfaults while cleaning up a h264_vulkan encoder that failed with "Encoding of h264 is not supported by this device", so the ladder never reached libopenh264 (#1340). Ask Vulkan for an H.264 encode queue family before avcodec_open2 and skip the rung when there is none.
1 parent 8f00d51 commit 63f248f

1 file changed

Lines changed: 95 additions & 0 deletions

File tree

‎electron/native/pipewire-capture/src/encoder.rs‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,98 @@ fn vaapi_is_safe_to_probe() -> bool {
177177
}
178178
}
179179

180+
/// True when the Vulkan device behind `device` has a queue family that encodes
181+
/// H.264 — the exact lookup `h264_vulkan` makes in `avcodec_open2` (issue #1340).
182+
///
183+
/// WHY ASK BEFORE OPENING. A driver can advertise VK_KHR_video_encode_queue on
184+
/// a GPU with no H.264 encode queue (RADV on an AMD Carrizo, in the report).
185+
/// ffmpeg 8.1 then fails with "Encoding of h264 is not supported by this device" and
186+
/// segfaults while freeing the half-built encoder, so the ladder never reaches
187+
/// libopenh264. Asking Vulkan first turns that crash into a skipped rung.
188+
///
189+
/// Read through Vulkan itself rather than `AVVulkanDeviceContext::qf`, because
190+
/// that struct needs the Vulkan headers to bind and the build does not have
191+
/// them. Only its first four pointer-sized fields are read: `alloc`,
192+
/// `get_proc_addr`, `inst` and `phys_dev`, in that order in the vendored 8.1.
193+
///
194+
/// SAFETY: `device` is a live AV_HWDEVICE_TYPE_VULKAN device context.
195+
unsafe fn vulkan_can_encode_h264(device: *mut ff::AVBufferRef) -> bool {
196+
use std::ffi::c_void;
197+
198+
#[repr(C)]
199+
struct DeviceHead {
200+
alloc: *const c_void,
201+
get_proc_addr: Option<GetInstanceProcAddr>,
202+
instance: *mut c_void,
203+
physical_device: *mut c_void,
204+
}
205+
#[repr(C)]
206+
struct QueueFamilyVideoProperties {
207+
s_type: i32,
208+
p_next: *mut c_void,
209+
codec_operations: u32,
210+
}
211+
#[repr(C)]
212+
struct QueueFamilyProperties2 {
213+
s_type: i32,
214+
p_next: *mut c_void,
215+
queue_flags: u32,
216+
queue_count: u32,
217+
timestamp_valid_bits: u32,
218+
min_image_transfer_granularity: [u32; 3],
219+
}
220+
type GetInstanceProcAddr =
221+
unsafe extern "C" fn(*mut c_void, *const std::ffi::c_char) -> *const c_void;
222+
type GetQueueFamilyProperties2 =
223+
unsafe extern "C" fn(*mut c_void, *mut u32, *mut QueueFamilyProperties2);
224+
225+
const STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2: i32 = 1_000_059_005;
226+
const STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR: i32 = 1_000_023_012;
227+
const QUEUE_VIDEO_ENCODE_BIT_KHR: u32 = 0x40;
228+
const VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR: u32 = 0x0001_0000;
229+
230+
let context = (*device).data as *const ff::AVHWDeviceContext;
231+
let head = (*context).hwctx as *const DeviceHead;
232+
let Some(get_proc_addr) = (*head).get_proc_addr else {
233+
return false;
234+
};
235+
let query = get_proc_addr(
236+
(*head).instance,
237+
c"vkGetPhysicalDeviceQueueFamilyProperties2".as_ptr(),
238+
);
239+
if query.is_null() {
240+
return false;
241+
}
242+
let query: GetQueueFamilyProperties2 = std::mem::transmute(query);
243+
244+
let mut count = 0u32;
245+
query((*head).physical_device, &mut count, ptr::null_mut());
246+
let mut video: Vec<QueueFamilyVideoProperties> = (0..count)
247+
.map(|_| QueueFamilyVideoProperties {
248+
s_type: STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
249+
p_next: ptr::null_mut(),
250+
codec_operations: 0,
251+
})
252+
.collect();
253+
let mut families: Vec<QueueFamilyProperties2> = video
254+
.iter_mut()
255+
.map(|video| QueueFamilyProperties2 {
256+
s_type: STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
257+
p_next: (video as *mut QueueFamilyVideoProperties).cast(),
258+
queue_flags: 0,
259+
queue_count: 0,
260+
timestamp_valid_bits: 0,
261+
min_image_transfer_granularity: [0; 3],
262+
})
263+
.collect();
264+
query((*head).physical_device, &mut count, families.as_mut_ptr());
265+
266+
families.iter().zip(&video).take(count as usize).any(|(family, video)| {
267+
family.queue_flags & QUEUE_VIDEO_ENCODE_BIT_KHR != 0
268+
&& video.codec_operations & VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR != 0
269+
})
270+
}
271+
180272
const RTLD_LAZY: i32 = 1;
181273
const RTLD_LOCAL: i32 = 0;
182274

@@ -497,6 +589,9 @@ impl VideoEncoder {
497589
ff::err_to_string(created)
498590
));
499591
}
592+
if device_type == ff::AV_HWDEVICE_TYPE_VULKAN && !vulkan_can_encode_h264(self.hw_device) {
593+
return Err("the Vulkan device has no H.264 encode queue".to_owned());
594+
}
500595

501596
self.hw_frames = ff::av_hwframe_ctx_alloc(self.hw_device);
502597
if self.hw_frames.is_null() {

0 commit comments

Comments
 (0)