Add fisheye projection support for skybox (#8577)

* feat: add fisheye projection support for skybox

Extends fisheye projection to the skybox so sky distortion matches
Gaussian splat distortion when using wide-angle/fisheye rendering.

Made-with: Cursor

* fix: gate SKY_FISHEYE define on SKYTYPE_INFINITE

Avoids compiling unused fisheye shader variants for box/dome sky types.

Made-with: Cursor

---------

Co-authored-by: Martin Valigursky <mvaligursky@snapchat.com>

Author: mvaligursky

examples/src/examples/gaussian-splatting/lod-streaming.controls.mjs

@@ -47,14 +47,27 @@ export const controls = ({ observer, ReactPCUI, React, jsx, fragment }) => {
                         { v: 'industrial-sunset', t: 'Industrial Sunset' },
                         { v: 'partly-cloudy', t: 'Partly Cloudy' },
                         { v: 'moonlit', t: 'Moonlit Sky' },
-                        { v: 'night-sky', t: 'Night Sky' }
+                        { v: 'sunflowers', t: 'Sunflowers' },
+                        { v: 'table-mountain', t: 'Table Mountain' },
+                        { v: 'cloud-layers', t: 'Cloud Layers' },
+                        { v: 'night', t: 'Night Sky' }
                     ]
                 })
             )
         ),
         jsx(
             Panel,
             { headerText: 'Camera' },
+            jsx(
+                LabelGroup,
+                { text: 'Camera Frame' },
+                jsx(BooleanInput, {
+                    type: 'toggle',
+                    binding: new BindingTwoWay(),
+                    link: { observer, path: 'cameraFrame' },
+                    value: observer.get('cameraFrame') || false
+                })
+            ),
             jsx(
                 LabelGroup,
                 { text: 'FOV' },

examples/src/examples/gaussian-splatting/lod-streaming.example.mjs

@@ -86,19 +86,31 @@ const ENV_PRESETS = {
     },
     'industrial-sunset': {
         url: 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/industrial_sunset_puresky_2k.hdr',
-        exposure: 0.3
+        exposure: 0.8
     },
     'partly-cloudy': {
         url: 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/kloofendal_48d_partly_cloudy_puresky_2k.hdr',
         exposure: 0.9
     },
     'moonlit': {
         url: 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/qwantani_moon_noon_puresky_2k.hdr',
-        exposure: 0.2
+        exposure: 0.4
+    },
+    'sunflowers': {
+        url: 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/sunflowers_puresky_2k.hdr',
+        exposure: 0.8
+    },
+    'table-mountain': {
+        url: 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/table_mountain_2_puresky_2k.hdr',
+        exposure: 1
+    },
+    'cloud-layers': {
+        url: 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/cloud_layers_2k.hdr',
+        exposure: 1
     },
-    'night-sky': {
+    'night': {
         url: 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/qwantani_night_puresky_2k.hdr',
-        exposure: 0.06
+        exposure: 0.2
     }
 };
 
@@ -224,18 +236,48 @@ assetListLoader.load(async () => {
         focusPoint: focusPoint
     });
 
+    // CameraFrame for HDR linear rendering (created lazily on first enable)
+    /** @type {pc.CameraFrame|null} */
+    let cameraFrame = null;
+
+    const applyToneMapping = () => {
+        const tm = data.get('toneMapping');
+        if (cameraFrame?.enabled) {
+            cameraFrame.rendering.toneMapping = tm;
+            cameraFrame.update();
+        } else {
+            camera.camera.toneMapping = tm;
+        }
+    };
+
+    data.set('cameraFrame', false);
+    data.on('cameraFrame:set', () => {
+        if (data.get('cameraFrame')) {
+            if (!cameraFrame) {
+                cameraFrame = new pc.CameraFrame(app, camera.camera);
+                cameraFrame.rendering.toneMapping = data.get('toneMapping');
+            }
+            cameraFrame.enabled = true;
+            cameraFrame.update();
+        } else if (cameraFrame) {
+            cameraFrame.destroy();
+            cameraFrame = null;
+        }
+        applyToneMapping();
+    });
+
     const applyFov = () => {
         const fov = data.get('cameraFov');
         camera.camera.fov = (data.get('fisheye') === 0) ? Math.min(fov, MAX_PERSPECTIVE_FOV) : fov;
     };
     data.on('cameraFov:set', applyFov);
     data.on('fisheye:set', () => {
-        app.scene.gsplat.fisheye = data.get('fisheye');
+        const fisheye = data.get('fisheye');
+        app.scene.gsplat.fisheye = fisheye;
+        app.scene.sky.fisheye = fisheye;
         applyFov();
     });
-    data.on('toneMapping:set', () => {
-        camera.camera.toneMapping = data.get('toneMapping');
-    });
+    data.on('toneMapping:set', applyToneMapping);
     data.on('exposure:set', () => {
         app.scene.exposure = data.get('exposure');
     });

src/scene/gsplat-unified/fisheye-projection.js

@@ -118,15 +118,20 @@ class FisheyeProjection {
         this.invK = 1.0 / k;
         this.cornerScale = 1.0 + (Math.SQRT2 - 1.0) * t;
 
-        // Projection-dependent values derived from the camera's projection matrix
-        const halfFovX = Math.atan2(1.0, p00);
+        // Projection-dependent values derived from the camera's projection matrix.
+        // Compute X and Y scales independently to avoid the 0/0 singularity at 180° FOV
+        // and to correctly handle the non-linear fisheye mapping for non-square aspect ratios.
         const maxTheta = Math.min(k * Math.PI / 2, 3.13);
-        const effHalfFov = Math.min(halfFovX, maxTheta - 0.01);
-        const gFov = k * Math.tan(effHalfFov / k);
-        const pm00 = this.cornerScale / gFov;
+        const cs = this.cornerScale;
+
+        const halfFovX = Math.atan2(1.0, p00);
+        const effHalfFovX = Math.min(halfFovX, maxTheta - 0.01);
+        this.projMat00 = cs / (k * Math.tan(effHalfFovX / k));
+
+        const halfFovY = Math.atan2(1.0, p11);
+        const effHalfFovY = Math.min(halfFovY, maxTheta - 0.01);
+        this.projMat11 = cs / (k * Math.tan(effHalfFovY / k));
 
-        this.projMat00 = pm00;
-        this.projMat11 = pm00 * p11 / p00;
         this.maxTheta = maxTheta;
     }
 }

src/scene/scene.js

@@ -770,7 +770,7 @@ class Scene extends EventHandler {
     }
 
     destroy() {
-        this._resetSkyMesh();
+        this._sky.destroy();
         this.root = null;
         this.off();
     }

src/scene/shader-lib/glsl/chunks/skybox/frag/skybox.js

@@ -13,6 +13,16 @@ export default /* glsl */`
     varying vec3 vViewDir;
     uniform float skyboxHighlightMultiplier;
 
+    #if defined(SKY_FISHEYE) && !defined(SKYMESH)
+        uniform float fisheye_k;
+        uniform float fisheye_invK;
+        uniform float fisheye_projMat00;
+        uniform float fisheye_projMat11;
+        uniform mat4 matrix_view;
+        uniform mat3 cubeMapRotationMatrix;
+        varying vec3 vClipXYW;
+    #endif
+
     #ifdef SKY_CUBEMAP
 
         uniform samplerCube texture_cubeMap;
@@ -42,28 +52,46 @@ export default /* glsl */`
 
         #else
 
-            #ifdef SKY_CUBEMAP
+            // --- compute view direction ---
+
+            #if defined(SKY_FISHEYE) && !defined(SKYMESH)
+
+                vec2 ndc = vClipXYW.xy / vClipXYW.z;
+                float px = ndc.x / fisheye_projMat00;
+                float py = ndc.y / fisheye_projMat11;
+                float r = sqrt(px * px + py * py);
+                float theta = fisheye_k * atan(r * fisheye_invK);
+                float sinT = sin(theta);
+                float cosT = cos(theta);
+                vec3 camDir = (r > 1e-6)
+                    ? vec3(px / r * sinT, py / r * sinT, -cosT)
+                    : vec3(0.0, 0.0, -1.0);
+                vec3 dir = transpose(mat3(matrix_view)) * camDir;
+                dir = dir * cubeMapRotationMatrix;
 
-                #ifdef SKYMESH
+            #elif defined(SKY_CUBEMAP) && defined(SKYMESH)
 
-                    // get vector from world space pos to tripod origin
-                    vec3 envDir = normalize(vWorldPos - projectedSkydomeCenter);
-                    vec3 dir = envDir * cubeMapRotationMatrix;
+                // get vector from world space pos to tripod origin
+                vec3 envDir = normalize(vWorldPos - projectedSkydomeCenter);
+                vec3 dir = envDir * cubeMapRotationMatrix;
 
-                #else
+            #else
 
-                    vec3 dir = vViewDir;
+                vec3 dir = vViewDir;
 
-                #endif
+            #endif
+
+            // --- sample environment ---
+
+            #ifdef SKY_CUBEMAP
 
                 dir.x *= -1.0;
                 vec3 linear = {SKYBOX_DECODE_FNC}(textureCube(texture_cubeMap, dir));
 
             #else // env-atlas
 
-                vec3 dir = vViewDir * vec3(-1.0, 1.0, 1.0);
+                dir *= vec3(-1.0, 1.0, 1.0);
                 vec2 uv = toSphericalUv(normalize(dir));
-
                 vec3 linear = {SKYBOX_DECODE_FNC}(texture2D(texture_envAtlas, mapRoughnessUv(uv, mipLevel)));
 
             #endif

src/scene/shader-lib/glsl/chunks/skybox/vert/skybox.js

@@ -7,6 +7,10 @@ uniform mat3 cubeMapRotationMatrix;
 
 varying vec3 vViewDir;
 
+#ifdef SKY_FISHEYE
+    varying vec3 vClipXYW;
+#endif
+
 #ifdef PREPASS_PASS
     // when skydome renders depth during prepass, generate linear depth
     varying float vLinearDepth;
@@ -35,9 +39,20 @@ void main(void) {
     #else
 
         view[3][0] = view[3][1] = view[3][2] = 0.0;
-        gl_Position = matrix_projectionSkybox * (view * aPosition);
         vViewDir = aPosition.xyz * cubeMapRotationMatrix;
 
+        #ifdef SKY_FISHEYE
+            // Bypass matrix_projectionSkybox which degenerates at extreme FOVs.
+            // Use a fixed ~90° perspective (p00=p11=1) so the box always covers the
+            // screen. The fragment shader recomputes view direction from screen
+            // coordinates, so only rasterization coverage matters here.
+            vec4 viewPos = view * aPosition;
+            gl_Position = vec4(viewPos.xy, 0.0, -viewPos.z);
+            vClipXYW = vec3(gl_Position.xy, gl_Position.w);
+        #else
+            gl_Position = matrix_projectionSkybox * (view * aPosition);
+        #endif
+
         #ifdef PREPASS_PASS
             // for infinite skybox, use negative gl_Position.w to get positive linear depth
             vLinearDepth = -gl_Position.w;

src/scene/shader-lib/wgsl/chunks/skybox/frag/skybox.js

@@ -14,6 +14,16 @@ export default /* wgsl */`
     varying vViewDir : vec3f;
     uniform skyboxHighlightMultiplier : f32;
 
+    #if defined(SKY_FISHEYE) && !defined(SKYMESH)
+        uniform fisheye_k : f32;
+        uniform fisheye_invK : f32;
+        uniform fisheye_projMat00 : f32;
+        uniform fisheye_projMat11 : f32;
+        uniform matrix_view : mat4x4f;
+        uniform cubeMapRotationMatrix : mat3x3f;
+        varying vClipXYW : vec3f;
+    #endif
+
     #ifdef SKY_CUBEMAP
 
         var texture_cubeMap : texture_cube<f32>;
@@ -52,22 +62,53 @@ export default /* wgsl */`
             var linear : vec3f;
             var dir : vec3f;
 
-            #ifdef SKY_CUBEMAP
+            // --- compute view direction ---
+
+            #if defined(SKY_FISHEYE) && !defined(SKYMESH)
+
+                let ndc : vec2f = input.vClipXYW.xy / input.vClipXYW.z;
+                let px : f32 = ndc.x / uniform.fisheye_projMat00;
+                let py : f32 = ndc.y / uniform.fisheye_projMat11;
+                let r : f32 = sqrt(px * px + py * py);
+                let theta : f32 = uniform.fisheye_k * atan(r * uniform.fisheye_invK);
+                let sinT : f32 = sin(theta);
+                let cosT : f32 = cos(theta);
+                var camDir : vec3f;
+                if (r > 1e-6) {
+                    camDir = vec3f(px / r * sinT, py / r * sinT, -cosT);
+                } else {
+                    camDir = vec3f(0.0, 0.0, -1.0);
+                }
+                let viewMat3 : mat3x3f = mat3x3f(
+                    uniform.matrix_view[0].xyz,
+                    uniform.matrix_view[1].xyz,
+                    uniform.matrix_view[2].xyz
+                );
+                dir = transpose(viewMat3) * camDir;
+                dir = dir * uniform.cubeMapRotationMatrix;
+
+            #elif defined(SKY_CUBEMAP) && defined(SKYMESH)
+
+                // get vector from world space pos to tripod origin
+                var envDir : vec3f = normalize(input.vWorldPos - uniform.projectedSkydomeCenter);
+                dir = envDir * uniform.cubeMapRotationMatrix;
+
+            #else
+
+                dir = input.vViewDir;
 
-                #ifdef SKYMESH
-                    // get vector from world space pos to tripod origin
-                    var envDir : vec3f = normalize(input.vWorldPos - uniform.projectedSkydomeCenter);
-                    dir = envDir * uniform.cubeMapRotationMatrix;
-                #else
-                    dir = input.vViewDir;
-                #endif
+            #endif
+
+            // --- sample environment ---
+
+            #ifdef SKY_CUBEMAP
 
                 dir.x *= -1.0;
                 linear = {SKYBOX_DECODE_FNC}(textureSample(texture_cubeMap, texture_cubeMap_sampler, dir));
 
             #else // env-atlas
 
-                dir = input.vViewDir * vec3f(-1.0, 1.0, 1.0);
+                dir *= vec3f(-1.0, 1.0, 1.0);
                 let uv : vec2f = toSphericalUv(normalize(dir));
                 linear = {SKYBOX_DECODE_FNC}(textureSample(texture_envAtlas, texture_envAtlas_sampler, mapRoughnessUv(uv, uniform.mipLevel)));
 

src/scene/shader-lib/wgsl/chunks/skybox/vert/skybox.js

@@ -8,6 +8,10 @@ export default /* wgsl */`
 
     varying vViewDir : vec3f;
 
+    #ifdef SKY_FISHEYE
+        varying vClipXYW : vec3f;
+    #endif
+
     #ifdef PREPASS_PASS
         // when skydome renders depth during prepass, generate linear depth
         varying vLinearDepth: f32;
@@ -40,9 +44,20 @@ export default /* wgsl */`
             view[3][0] = 0.0;
             view[3][1] = 0.0;
             view[3][2] = 0.0;
-            output.position = uniform.matrix_projectionSkybox * (view * input.aPosition);
             output.vViewDir = input.aPosition.xyz * uniform.cubeMapRotationMatrix;
 
+            #ifdef SKY_FISHEYE
+                // Bypass matrix_projectionSkybox which degenerates at extreme FOVs.
+                // Use a fixed ~90° perspective (p00=p11=1) so the box always covers the
+                // screen. The fragment shader recomputes view direction from screen
+                // coordinates, so only rasterization coverage matters here.
+                var viewPos : vec4f = view * input.aPosition;
+                output.position = vec4f(viewPos.xy, 0.0, -viewPos.z);
+                output.vClipXYW = vec3f(output.position.xy, output.position.w);
+            #else
+                output.position = uniform.matrix_projectionSkybox * (view * input.aPosition);
+            #endif
+
             #ifdef PREPASS_PASS
                 // for infinite skybox, use negative gl_Position.w to get positive linear depth
                 output.vLinearDepth = -pcPosition.w;