Skip to content

Commit c85e3c7

Browse files
committed
feat(effects): footage() reads the recording at its own resolution
frame() holds the composed picture at the output's resolution, so an effect that magnifies (a loupe, a lens) only magnified blur. footage(p) reads the recording itself at point p of its surface, the space of fx_to_footage, transparent outside 0..1. Each backend paints the content cut into an RGBA copy at the decoder texture's resolution (at most 4096 a side) with one mode-0 draw, the same YUV conversion as the drawn screen, and binds it at t4 / texture(1) / binding 4. analyze() tells from naga's global usage whether fx_fs reads fx_footage; reads_footage() memoises it per source, so an effect that does not read it pays nothing. footage() reads count in the cost tier like frame() reads.
1 parent 2159f42 commit c85e3c7

8 files changed

Lines changed: 552 additions & 34 deletions

File tree

‎crates/compositor/src/compositor_linux.rs‎

Lines changed: 127 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -415,8 +415,12 @@ pub struct Compositor {
415415
/// Pyramide de profondeur de champ (mode 8) : la video en RGBA8 a demi-resolution
416416
/// de la texture DECODEUR, mips compris (texture, vue tous niveaux, vue par niveau),
417417
/// et sa taille de niveau 0. Allouee au premier ecran incline, recreee quand la
418-
/// texture decodeur change de taille. Cf. `compositor_windows::DofPyramid`.
418+
/// texture decodeur change de taille. Cf. `compositor_windows::RgbaTarget`.
419419
dof_pyramid: RefCell<Option<DofPyramid>>,
420+
/// L'enregistrement a sa resolution, pour les effets qui lisent `footage()` (taille, texture,
421+
/// vue) : alloue au premier d'entre eux, recree quand la coupe change de taille, peint
422+
/// seulement aux frames ou un tel effet est actif. Cf. `compositor_windows::fill_footage_copy`.
423+
footage_copy: RefCell<Option<(u32, u32, wgpu::Texture, wgpu::TextureView)>>,
420424

421425
/// Images d'annotation, indexees par ID d'annotation -- PAS par chemin comme
422426
/// `img_cache`. Une annotation image porte souvent une data-URI de plusieurs
@@ -758,6 +762,14 @@ impl Compositor {
758762
fx_entry(1, wgpu::BindingType::Sampler(wgpu::SamplerBindingType::Filtering)),
759763
fx_entry(2, fx_uniform),
760764
fx_entry(3, fx_uniform),
765+
fx_entry(
766+
4,
767+
wgpu::BindingType::Texture {
768+
sample_type: wgpu::TextureSampleType::Float { filterable: true },
769+
view_dimension: wgpu::TextureViewDimension::D2,
770+
multisampled: false,
771+
},
772+
),
761773
],
762774
});
763775
let shader_fx_layout = gpu.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor {
@@ -828,6 +840,7 @@ impl Compositor {
828840
ann_copy_view,
829841
ann_copy_mips,
830842
dof_pyramid: RefCell::new(None),
843+
footage_copy: RefCell::new(None),
831844
ann_img_cache: RefCell::new(std::collections::HashMap::new()),
832845
webcam_mask: RefCell::new(None),
833846
seg_capture: RefCell::new(None),
@@ -1399,6 +1412,7 @@ impl Compositor {
13991412
effect: &SceneShaderEffect,
14001413
dst: (&wgpu::Texture, &wgpu::TextureView),
14011414
ctx: &crate::shader_fx::FxContext,
1415+
footage: Option<&wgpu::TextureView>,
14021416
) {
14031417
let Some(pipeline) = self.shader_fx_ready(effect) else { return };
14041418
let ctx_bytes = unsafe {
@@ -1417,7 +1431,10 @@ impl Compositor {
14171431
})
14181432
};
14191433
let (ctx_buf, params_buf) = (uniform("shader-fx-ctx", ctx_bytes), uniform("shader-fx-params", &params));
1420-
// Les bindings du prélude : la frame (0), le sampler (1), `ctx` (2), `params` (3).
1434+
// Les bindings du prélude : la frame (0), le sampler (1), `ctx` (2), `params` (3) et
1435+
// `footage()` (4), une texture vide pour un effet qui ne la lit pas.
1436+
let dummy = footage.is_none().then(|| self.dummy_view());
1437+
let footage = footage.or(dummy.as_ref()).expect("une vue ou l'autre");
14211438
let bind = self.gpu.device.create_bind_group(&wgpu::BindGroupDescriptor {
14221439
label: Some("shader-fx"),
14231440
layout: &self.shader_fx_bgl,
@@ -1432,6 +1449,7 @@ impl Compositor {
14321449
},
14331450
wgpu::BindGroupEntry { binding: 2, resource: ctx_buf.as_entire_binding() },
14341451
wgpu::BindGroupEntry { binding: 3, resource: params_buf.as_entire_binding() },
1452+
wgpu::BindGroupEntry { binding: 4, resource: wgpu::BindingResource::TextureView(footage) },
14351453
],
14361454
});
14371455
encoder.copy_texture_to_texture(
@@ -1455,6 +1473,66 @@ impl Compositor {
14551473
rpass.draw(0..3, 0..1);
14561474
}
14571475

1476+
/// Peint l'enregistrement a sa resolution pour `footage()` (`docs/shader-effects.md`, C.2). Port
1477+
/// de `compositor_windows::fill_footage_copy` : la coupe du metrage (`content_cut`, le repere de
1478+
/// `fx_to_footage`) en UN draw du mode 0, par la meme conversion que l'ecran dessine.
1479+
#[allow(clippy::too_many_arguments)]
1480+
fn encode_footage_copy(
1481+
&self,
1482+
encoder: &mut wgpu::CommandEncoder,
1483+
planes: (&wgpu::TextureView, &wgpu::TextureView, &wgpu::TextureView),
1484+
cut: [f32; 4],
1485+
tex_w: u32,
1486+
tex_h: u32,
1487+
) -> wgpu::TextureView {
1488+
let (w, h) = crate::shader_fx::footage_copy_px(cut, [tex_w as f32, tex_h as f32]);
1489+
let stale = self.footage_copy.borrow().as_ref().is_none_or(|c| (c.0, c.1) != (w, h));
1490+
if stale {
1491+
let tex = self.gpu.device.create_texture(&wgpu::TextureDescriptor {
1492+
label: Some("footage-copy"),
1493+
size: wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
1494+
mip_level_count: 1,
1495+
sample_count: 1,
1496+
dimension: wgpu::TextureDimension::D2,
1497+
format: wgpu::TextureFormat::Rgba8Unorm,
1498+
usage: wgpu::TextureUsages::RENDER_ATTACHMENT | wgpu::TextureUsages::TEXTURE_BINDING,
1499+
view_formats: &[],
1500+
});
1501+
let view = tex.create_view(&wgpu::TextureViewDescriptor::default());
1502+
*self.footage_copy.borrow_mut() = Some((w, h, tex, view));
1503+
}
1504+
let view = self.footage_copy.borrow().as_ref().expect("copie allouee ci-dessus").3.clone();
1505+
let full = [0.0, 0.0, 1.0, 1.0];
1506+
let cb = LayerCB {
1507+
dst: full,
1508+
src: cut,
1509+
quad_px: [w as f32, h as f32],
1510+
mode: 0.0,
1511+
color: [1.0, 1.0, 1.0, 1.0],
1512+
src_prev: cut,
1513+
dst_prev: full,
1514+
mb: [1.0, 0.0, 1.0, 0.0],
1515+
..Default::default()
1516+
};
1517+
let dummy = self.dummy_view();
1518+
let fill = self.make_bind(&cb, Some(planes), &dummy);
1519+
let mut rpass = encoder.begin_render_pass(&wgpu::RenderPassDescriptor {
1520+
label: Some("footage-copy-pass"),
1521+
color_attachments: &[Some(wgpu::RenderPassColorAttachment {
1522+
view: &view,
1523+
resolve_target: None,
1524+
ops: wgpu::Operations { load: wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT), store: wgpu::StoreOp::Store },
1525+
})],
1526+
depth_stencil_attachment: None,
1527+
timestamp_writes: None,
1528+
occlusion_query_set: None,
1529+
});
1530+
rpass.set_pipeline(&self.pipeline);
1531+
self.draw_layer(&mut rpass, &fill, false);
1532+
drop(rpass);
1533+
view
1534+
}
1535+
14581536
/// Le calque transparent des textes, à la taille de rendu, alloué à la première frame qui en a
14591537
/// besoin (un effet `ShaderEffectTarget::Text`).
14601538
fn text_layer(&self) -> (wgpu::Texture, wgpu::TextureView) {
@@ -3765,8 +3843,13 @@ impl Compositor {
37653843
let with_target = |target: ShaderEffectTarget| -> Vec<&SceneShaderEffect> {
37663844
effects.iter().copied().filter(|e| e.target == target).collect()
37673845
};
3846+
// L'enregistrement a sa resolution, peint une fois pour la frame si un effet le lit.
3847+
let footage = effects
3848+
.iter()
3849+
.any(|e| crate::shader_fx::reads_footage(e))
3850+
.then(|| self.encode_footage_copy(&mut encoder, (&sy, &su, &sv), g.content_cut, stw, sth));
37683851
for e in with_target(ShaderEffectTarget::Image) {
3769-
self.encode_shader_effect(&mut encoder, e, (&self.rt, &self.rt_view), &fx_context(e));
3852+
self.encode_shader_effect(&mut encoder, e, (&self.rt, &self.rt_view), &fx_context(e), footage.as_ref());
37703853
}
37713854
// Un effet `text` : les textes sur un calque transparent, l'effet sur ce calque seul, puis
37723855
// le calque pose sur l'image (« over » premultiplie). L'image dessous reste intacte.
@@ -3800,7 +3883,7 @@ impl Compositor {
38003883
}
38013884
if let Some((tex, view)) = &text_layer {
38023885
for e in text_effects {
3803-
self.encode_shader_effect(&mut encoder, e, (tex, view), &fx_context(e));
3886+
self.encode_shader_effect(&mut encoder, e, (tex, view), &fx_context(e), footage.as_ref());
38043887
}
38053888
// Le calque pose sur le RT par la passe qui recopie `accum` : layout du blur, et un
38063889
// uniforme que `fs_copy` ne lit pas.
@@ -3837,7 +3920,7 @@ impl Compositor {
38373920
rpass.draw(0..3, 0..1);
38383921
}
38393922
for e in with_target(ShaderEffectTarget::Both) {
3840-
self.encode_shader_effect(&mut encoder, e, (&self.rt, &self.rt_view), &fx_context(e));
3923+
self.encode_shader_effect(&mut encoder, e, (&self.rt, &self.rt_view), &fx_context(e), footage.as_ref());
38413924
}
38423925
self.gpu.context.submit(std::iter::once(encoder.finish()));
38433926
Ok(())
@@ -6993,6 +7076,45 @@ mod tests {
69937076
Some(corner_at(&fx_compositor(&gpu, 320, 180, effects), &gpu, 2.0))
69947077
}
69957078

7079+
/// `footage()` lit l'enregistrement à sa résolution, pas l'image composée : la source de 512 px
7080+
/// en colonnes d'un pixel est réduite à ~90 px sous le padding, où l'image composée ne les sépare
7081+
/// plus, mais chacune se lit blanche ou noire. Hors de 0..1, rien. Cf. compositor_windows.
7082+
#[test]
7083+
fn footage_reads_the_recording_at_its_own_resolution() {
7084+
let Some(gpu) = gpu() else { return };
7085+
let pixel = |code: &str| {
7086+
let comp = fx_compositor(&gpu, 320, 180, serde_json::json!([fx(code, "image")]));
7087+
let screen = FakeFrame::new(&gpu, 512, 512, |x, _| if x % 2 == 0 { 235 } else { 16 });
7088+
comp.set_timeline_time(Some(2.0));
7089+
comp.set_programme_time(Some(2.0));
7090+
unsafe { comp.compose_frame(screen.as_ptr(), std::ptr::null(), 0.0, &fx_cfg()) }.expect("compose");
7091+
let (_, _, rgba) = unsafe { comp.readback_direct() }.expect("readback");
7092+
[rgba[0], rgba[1], rgba[2], rgba[3]]
7093+
};
7094+
let at = |x: &str| format!("fn effect(uv: vec2<f32>) -> vec4<f32> {{ return footage(vec2<f32>({x}, 0.5)); }}");
7095+
let near = |got: [u8; 4], want: [u8; 4]| got.iter().zip(want).all(|(g, w)| g.abs_diff(w) <= 2);
7096+
let white = pixel(&at("10.5 / 512.0"));
7097+
let black = pixel(&at("11.5 / 512.0"));
7098+
assert!(near(white, [255, 255, 255, 255]), "colonne 10 : {white:?}");
7099+
assert!(near(black, [0, 0, 0, 255]), "colonne 11 : {black:?}");
7100+
assert_eq!(pixel(&at("1.5")), [0, 0, 0, 0], "hors de l'enregistrement");
7101+
let composed = |x: &str| {
7102+
pixel(&format!("fn effect(uv: vec2<f32>) -> vec4<f32> {{ return frame(fx_from_footage(vec2<f32>({x}, 0.5))); }}"))
7103+
};
7104+
let (left, right) = (composed("10.5 / 512.0"), composed("11.5 / 512.0"));
7105+
assert!(left[0].abs_diff(right[0]) < 128, "l'image composée sépare les colonnes : {left:?} {right:?}");
7106+
}
7107+
7108+
/// La copie de l'enregistrement coûte une passe : un effet qui ne lit pas `footage()` ne la
7109+
/// fait jamais faire.
7110+
#[test]
7111+
fn an_effect_that_does_not_read_the_footage_never_paints_it() {
7112+
let Some(gpu) = gpu() else { return };
7113+
let comp = fx_compositor(&gpu, 320, 180, serde_json::json!([fx(INVERT, "image")]));
7114+
corner_at(&comp, &gpu, 2.0);
7115+
assert!(comp.footage_copy.borrow().is_none());
7116+
}
7117+
69967118
/// La passe de l'agent repeint l'image composée, dans sa région et nulle part ailleurs. Le
69977119
/// compositeur par défaut est celui de l'export : il attend le pipeline, jamais de frame sans.
69987120
#[test]

0 commit comments

Comments
 (0)