Skip to content

Tune quickstart capture framing and downsampling for documentation media #7584

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
mataylor-nvidia wants to merge 1 commit into
base: develop
Choose a base branch
from
+27 −18
Draft

Tune quickstart capture framing and downsampling for documentation media #7584

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions tools/docs/media/capture_quickstart.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
from isaaclab_tasks.core.velocity.config.g1.flat_env_cfg import G1FlatEnvCfg
from isaaclab_tasks.utils import resolve_task_config, setup_preset_cli

# Focal lengths are tuned per task to the tightest framing that still keeps the subject inside the
# published crop; the safe value differs per scene, so they are not a fixed ratio of one another.
_TASK_CONFIGS = {
"Isaac-Cartpole": "capture_quickstart:CartpoleCaptureCfg",
"Isaac-Lift-KukaAllegro": "capture_quickstart:KukaAllegroCaptureCfg",
Expand All @@ -43,7 +45,7 @@ class CartpoleCaptureCfg(CartpoleEnvCfg):

def __post_init__(self):
super().__post_init__()
_configure_capture(self, focal_length=18.0)
_configure_capture(self, focal_length=21.0)


@configclass
Expand All @@ -52,7 +54,7 @@ class G1FlatCaptureCfg(G1FlatEnvCfg):

def __post_init__(self):
super().__post_init__()
_configure_capture(self, focal_length=20.0)
_configure_capture(self, focal_length=29.0)


@configclass
Expand All @@ -61,7 +63,9 @@ class KukaAllegroCaptureCfg(KukaAllegroLiftEnvCfg):

def __post_init__(self):
super().__post_init__()
_configure_capture(self, focal_length=22.0)
# Held to 24 rather than the tighter framing the other tasks take: the lift policy raises
# the arm well above the pose a zero action holds, and 26 already leaves little headroom.
_configure_capture(self, focal_length=24.0)


@configclass
Expand All @@ -70,7 +74,7 @@ class FrankaCabinetCaptureCfg(FrankaCabinetEnvCfg):

def __post_init__(self):
super().__post_init__()
_configure_capture(self, focal_length=22.0)
_configure_capture(self, focal_length=32.0)


def configure_playback() -> list[str]:
Expand All @@ -97,7 +101,7 @@ def main():
env_cfg, _ = resolve_task_config(args.task, "", play_mode=True)
env_cfg.scene.num_envs = 1
env_cfg.seed = 42
_configure_resolved_sim(env_cfg.sim, focal_length=22.0)
_configure_resolved_sim(env_cfg.sim, focal_length=32.0)
env_cfg.video_recorders = [
VideoRecorderCfg(
source="visualizer:newton_rtx",
Expand Down Expand Up @@ -144,15 +148,20 @@ def _configure_capture(env_cfg: object, focal_length: float):


def _configure_resolved_sim(sim_cfg: SimulationCfg, focal_length: float):
"""Attach a headless 320 by 240 OVRTX visualizer to a simulation config."""
"""Attach a headless 1280 by 960 OVRTX visualizer to a simulation config.

Captures at four times the 320 by 240 publication size so the generator can downsample the
path-traced frames, which resolves robot silhouettes and shadow edges that alias away when
the path tracer renders straight to the final size.
"""
default_camera = sim_cfg.default_visualizer_cfg or VisualizerCfg()
sim_cfg.visualizer_cfgs = [
NewtonRTXVisualizerCfg(
eye=default_camera.eye,
lookat=default_camera.lookat,
focal_length=focal_length,
window_width=320,
window_height=240,
window_width=1280,
window_height=960,
headless=True,
rtx_environment="default",
render_settings={"omni:rtx:quality": ("Int", 100)},
Expand Down
20 changes: 10 additions & 10 deletions tools/docs/media/generate_quickstart.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,25 +75,25 @@ ffmpeg -y -v error \
-i "${WORK_DIR}/random/random_0000.mp4" \
-i "${WORK_DIR}/play/play_0000.mp4" \
-filter_complex \
"[0:v]select='not(mod(n,5))',setpts=N/(12*TB),hqdn3d=2:2:6:6,drawbox=x=8:y=8:w=112:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='zero_agent':fontcolor=white:fontsize=16:x=16:y=13[zero];
[1:v]select='not(mod(n,5))',setpts=N/(12*TB),hqdn3d=2:2:6:6,drawbox=x=8:y=8:w=134:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='random_agent':fontcolor=white:fontsize=16:x=16:y=13[random];
[2:v]select='not(mod(n,5))',setpts=N/(12*TB),hqdn3d=2:2:6:6,drawbox=x=8:y=8:w=58:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='play':fontcolor=white:fontsize=16:x=16:y=13[play];
"[0:v]select='not(mod(n,5))',setpts=N/(12*TB),scale=320:240:flags=lanczos,hqdn3d=1:1:3:3,drawbox=x=8:y=8:w=112:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='zero_agent':fontcolor=white:fontsize=16:x=16:y=13[zero];
[1:v]select='not(mod(n,5))',setpts=N/(12*TB),scale=320:240:flags=lanczos,hqdn3d=1:1:3:3,drawbox=x=8:y=8:w=134:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='random_agent':fontcolor=white:fontsize=16:x=16:y=13[random];
[2:v]select='not(mod(n,5))',setpts=N/(12*TB),scale=320:240:flags=lanczos,hqdn3d=1:1:3:3,drawbox=x=8:y=8:w=58:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='play':fontcolor=white:fontsize=16:x=16:y=13[play];
[zero][random][play]hstack=inputs=3,split[frames][palette_frames];
[palette_frames]palettegen=max_colors=128:stats_mode=diff[palette];
[frames][palette]paletteuse=dither=bayer:bayer_scale=4:diff_mode=rectangle" \
[palette_frames]palettegen=max_colors=256:stats_mode=diff[palette];
[frames][palette]paletteuse=dither=sierra2_4a:diff_mode=rectangle" \
-loop 0 "${OUTPUT_DIR}/agent-comparison.gif"

ffmpeg -y -v error \
-i "${WORK_DIR}/cartpole/cartpole_0000.mp4" \
-i "${WORK_DIR}/g1/g1_0000.mp4" \
-i "${WORK_DIR}/kuka_allegro/kuka_allegro_0000.mp4" \
-filter_complex \
"[0:v]crop=320:180:0:30,select='not(mod(n,5))',setpts=N/(12*TB),hqdn3d=2:2:6:6,drawbox=x=8:y=8:w=82:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='Cartpole':fontcolor=white:fontsize=16:x=16:y=13[cartpole];
[1:v]crop=320:180:0:30,select='not(mod(n,5))',setpts=N/(12*TB),hqdn3d=2:2:6:6,drawbox=x=8:y=8:w=128:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='G1 locomotion':fontcolor=white:fontsize=16:x=16:y=13[g1];
[2:v]crop=320:180:0:30,select='not(mod(n,5))',setpts=N/(12*TB),hqdn3d=2:2:6:6,drawbox=x=8:y=8:w=116:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='Kuka Allegro':fontcolor=white:fontsize=16:x=16:y=13[kuka];
"[0:v]crop=1280:720:0:120,select='not(mod(n,5))',setpts=N/(12*TB),scale=320:180:flags=lanczos,hqdn3d=1:1:3:3,drawbox=x=8:y=8:w=82:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='Cartpole':fontcolor=white:fontsize=16:x=16:y=13[cartpole];
[1:v]crop=1280:720:0:120,select='not(mod(n,5))',setpts=N/(12*TB),scale=320:180:flags=lanczos,hqdn3d=1:1:3:3,drawbox=x=8:y=8:w=128:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='G1 locomotion':fontcolor=white:fontsize=16:x=16:y=13[g1];
[2:v]crop=1280:720:0:120,select='not(mod(n,5))',setpts=N/(12*TB),scale=320:180:flags=lanczos,hqdn3d=1:1:3:3,drawbox=x=8:y=8:w=116:h=28:color=black@0.65:t=fill,drawtext=font='DejaVu Sans':text='Kuka Allegro':fontcolor=white:fontsize=16:x=16:y=13[kuka];
[cartpole][g1][kuka]hstack=inputs=3,split[frames][palette_frames];
[palette_frames]palettegen=max_colors=128:stats_mode=diff[palette];
[frames][palette]paletteuse=dither=bayer:bayer_scale=4:diff_mode=rectangle" \
[palette_frames]palettegen=max_colors=256:stats_mode=diff[palette];
[frames][palette]paletteuse=dither=sierra2_4a:diff_mode=rectangle" \
-loop 0 "${OUTPUT_DIR}/task-sampler.gif"

for output in agent-comparison.gif task-sampler.gif; do
Expand Down
Loading