Support non-pinhole rendering (#1141)

nerfstudio/cameras/camera_paths.py

@@ -23,7 +23,7 @@ import torch
 import nerfstudio.utils.poses as pose_utils
 from nerfstudio.cameras import camera_utils
 from nerfstudio.cameras.camera_utils import get_interpolated_poses_many
-from nerfstudio.cameras.cameras import Cameras
+from nerfstudio.cameras.cameras import Cameras, CameraType
 from nerfstudio.viewer.server.utils import three_js_perspective_camera_focal_length
 
 
@@ -121,6 +121,19 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
 
     image_height = camera_path["render_height"]
     image_width = camera_path["render_width"]
+    if "camera_type" in camera_path:
+        camera_type = camera_path["camera_type"]
+    else:
+        camera_type = "perspective"
+
+    if "camera_type" not in camera_path:
+        camera_type = CameraType.PERSPECTIVE
+    elif camera_path["camera_type"] == "fisheye":
+        camera_type = CameraType.FISHEYE
+    elif camera_path["camera_type"] == "equirectangular":
+        camera_type = CameraType.EQUIRECTANGULAR
+    else:
+        camera_type = CameraType.PERSPECTIVE
 
     c2ws = []
     fxs = []
@@ -129,11 +142,15 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
         # pose
         c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3]
         c2ws.append(c2w)
-        # field of view
-        fov = camera["fov"]
-        focal_length = three_js_perspective_camera_focal_length(fov, image_height)
-        fxs.append(focal_length)
-        fys.append(focal_length)
+        if camera_type == CameraType.EQUIRECTANGULAR:
+            fxs.append(image_width / 2)
+            fys.append(image_height)
+        else:
+            # field of view
+            fov = camera["fov"]
+            focal_length = three_js_perspective_camera_focal_length(fov, image_height)
+            fxs.append(focal_length)
+            fys.append(focal_length)
 
     camera_to_worlds = torch.stack(c2ws, dim=0)
     fx = torch.tensor(fxs)
@@ -144,4 +161,5 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
         cx=image_width / 2,
         cy=image_height / 2,
         camera_to_worlds=camera_to_worlds,
+        camera_type=camera_type,
     )

nerfstudio/cameras/cameras.py

@@ -634,7 +634,7 @@ class Cameras(TensorDataclass):
 
             directions_stack[..., 0][mask] = torch.masked_select(coord_stack[..., 0] * sin_theta / theta, mask).float()
             directions_stack[..., 1][mask] = torch.masked_select(coord_stack[..., 1] * sin_theta / theta, mask).float()
-            directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask)
+            directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask).float()
 
         if CameraType.EQUIRECTANGULAR.value in cam_types:
             mask = (self.camera_type[true_indices] == CameraType.EQUIRECTANGULAR.value).squeeze(-1)  # (num_rays)

nerfstudio/viewer/app/package.json

@@ -1,7 +1,7 @@
 {
   "name": "viewer",
   "homepage": ".",
-  "version": "22-12-02-0",
+  "version": "22-12-16-0",
   "private": true,
   "dependencies": {
     "@emotion/react": "^11.10.4",

nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx

@@ -112,12 +112,12 @@ function FovSelector(props) {
         camera.setFocalLength(val / camera.aspect);
       }
     } else if (fovLabel === FOV_LABELS.FOV) {
+      camera.fov = val;
       setGlobalFov(val);
     } else {
-      const old_fov = camera.fov;
       camera.setFocalLength(val / camera.aspect);
       const new_fov = camera.getEffectiveFOV();
-      camera.fov = old_fov;
+      camera.fov = new_fov;
       setGlobalFov(new_fov);
     }
 
@@ -216,6 +216,8 @@ function CameraList(props) {
   const [slider_value, set_slider_value] = React.useState(0);
   const [expanded, setExpanded] = React.useState(null);
 
+  const camera_type = useSelector((state) => state.renderingState.camera_type);
+
   const handleChange =
     (cameraUUID: string) =>
     (event: React.SyntheticEvent, isExpanded: boolean) => {
@@ -399,7 +401,7 @@ function CameraList(props) {
           </Stack>
         </AccordionSummary>
         <AccordionDetails>
-          {isAnimated('FOV') && (
+          {isAnimated('FOV') && camera_type !== 'equirectangular' && (
             <FovSelector
               fovLabel={fovLabel}
               setFovLabel={setFovLabel}
@@ -485,6 +487,7 @@ export default function CameraPanel(props) {
   const render_width = useSelector(
     (state) => state.renderingState.render_width,
   );
+  const camera_type = useSelector((state) => state.renderingState.camera_type);
 
   const setRenderHeight = (value) => {
     dispatch({
@@ -501,6 +504,22 @@ export default function CameraPanel(props) {
     });
   };
 
+  const setCameraType = (value) => {
+    dispatch({
+      type: 'write',
+      path: 'renderingState/camera_type',
+      data: value,
+    });
+  };
+
+  const setFieldOfView = (value) => {
+    dispatch({
+      type: 'write',
+      path: 'renderingState/field_of_view',
+      data: value,
+    });
+  };
+
   // ui state
   const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV);
 
@@ -516,7 +535,7 @@ export default function CameraPanel(props) {
     // set slider and render camera back to 0
     if (new_camera_list.length >= 1) {
       set_camera_position(camera_render, new_camera_list[0].matrix);
-      camera_render.fov = new_camera_list[0].fov;
+      setFieldOfView(new_camera_list[0].fov);
       set_slider_value(slider_min);
     }
   };
@@ -718,7 +737,7 @@ export default function CameraPanel(props) {
 
     const mat = get_transform_matrix(position, lookat, up);
     set_camera_position(camera_render, mat);
-    camera_render.fov = fov;
+    setFieldOfView(fov);
   } else {
     sceneTree.delete(['Camera Path', 'Curve']);
   }
@@ -768,7 +787,7 @@ export default function CameraPanel(props) {
       fov = curve_object.curve_fovs.getPoint(point).z;
       const mat = get_transform_matrix(position, lookat, up);
       set_camera_position(camera_render, mat);
-      camera_render.fov = fov;
+      setFieldOfView(fov);
       setGlobalFov(fov);
     }
   }, [slider_value, render_height, render_width]);
@@ -828,6 +847,7 @@ export default function CameraPanel(props) {
     // const myData
     const camera_path_object = {
       keyframes,
+      camera_type,
       render_height,
       render_width,
       camera_path,
@@ -871,6 +891,7 @@ export default function CameraPanel(props) {
 
     setRenderHeight(camera_path_object.render_height);
     setRenderWidth(camera_path_object.render_width);
+    setCameraType(camera_path_object.camera_type);
 
     setFps(camera_path_object.fps);
     setSeconds(camera_path_object.seconds);
@@ -1024,52 +1045,54 @@ export default function CameraPanel(props) {
           />
         </LevaStoreProvider>
       </div>
-      <div className="CameraList-row-animation-properties">
-        <Tooltip title="Animate FOV for Each Camera">
-          <Button
-            value="animatefov"
-            selected={isAnimated('FOV')}
-            onClick={() => {
-              toggleAnimate('FOV');
-            }}
-            style={{
-              maxWidth: '20px',
-              maxHeight: '20px',
-              minWidth: '20px',
-              minHeight: '20px',
-              position: 'relative',
-              top: '22px',
-            }}
-            sx={{
-              mt: 1,
-            }}
-          >
-            <Animation
+      {camera_type !== 'equirectangular' && (
+        <div className="CameraList-row-animation-properties">
+          <Tooltip title="Animate FOV for Each Camera">
+            <Button
+              value="animatefov"
+              selected={isAnimated('FOV')}
+              onClick={() => {
+                toggleAnimate('FOV');
+              }}
               style={{
-                color: isAnimated('FOV') ? '#24B6FF' : '#EBEBEB',
                 maxWidth: '20px',
                 maxHeight: '20px',
                 minWidth: '20px',
                 minHeight: '20px',
+                position: 'relative',
+                top: '22px',
               }}
-            />
-          </Button>
-        </Tooltip>
-        <FovSelector
-          fovLabel={fovLabel}
-          setFovLabel={setFovLabel}
-          camera={camera_main}
-          cameras={cameras}
-          dispatch={dispatch}
-          disabled={isAnimated('FOV')}
-          applyAll={!isAnimated('FOV')}
-          isGlobal
-          globalFov={globalFov}
-          setGlobalFov={setGlobalFov}
-          setAllCameraFOV={setAllCameraFOV}
-          changeMain
-        />
-      </div>
+              sx={{
+                mt: 1,
+              }}
+            >
+              <Animation
+                style={{
+                  color: isAnimated('FOV') ? '#24B6FF' : '#EBEBEB',
+                  maxWidth: '20px',
+                  maxHeight: '20px',
+                  minWidth: '20px',
+                  minHeight: '20px',
+                }}
+              />
+            </Button>
+          </Tooltip>
+          <FovSelector
+            fovLabel={fovLabel}
+            setFovLabel={setFovLabel}
+            camera={camera_main}
+            cameras={cameras}
+            dispatch={dispatch}
+            disabled={isAnimated('FOV')}
+            applyAll={!isAnimated('FOV')}
+            isGlobal
+            globalFov={globalFov}
+            setGlobalFov={setGlobalFov}
+            setAllCameraFOV={setAllCameraFOV}
+            changeMain
+          />
+        </div>
+      )}
       <div>
         <div className="CameraPanel-row">
           <Button

nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPropPanel.jsx

@@ -18,6 +18,7 @@ export default function CameraPropPanel(props) {
   const render_width = useSelector(
     (state) => state.renderingState.render_width,
   );
+  const camera_type = useSelector((state) => state.renderingState.camera_type);
 
   const setResolution = (value) => {
     dispatch({
@@ -32,6 +33,14 @@ export default function CameraPropPanel(props) {
     });
   };
 
+  const setCameraType = (value) => {
+    dispatch({
+      type: 'write',
+      path: 'renderingState/camera_type',
+      data: value,
+    });
+  };
+
   const [, setControls] = useControls(
     () => ({
       camera_resolution: {
@@ -59,6 +68,18 @@ export default function CameraPropPanel(props) {
           set_fps(v);
         },
       },
+      camera_type_selector: {
+        label: 'Camera Type',
+        value: camera_type,
+        options: {
+          Perspective: 'perspective',
+          Fisheye: 'fisheye',
+          Equirectangular: 'equirectangular',
+        },
+        onChange: (v) => {
+          setCameraType(v);
+        },
+      },
     }),
     { store },
   );
@@ -68,6 +89,7 @@ export default function CameraPropPanel(props) {
   setControls({
     camera_resolution: { width: render_width, height: render_height },
   });
+  setControls({ camera_type_selector: camera_type });
 
   return null;
 }

nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx

@@ -116,6 +116,7 @@ export default function ViewerWindow(props) {
   const camera_choice = useSelector(
     (state) => state.renderingState.camera_choice,
   );
+  const camera_type = useSelector((state) => state.renderingState.camera_type);
 
   // listen to the viewport width
   const size = new THREE.Vector2();
@@ -148,21 +149,16 @@ export default function ViewerWindow(props) {
     labelRenderer.setSize(viewportWidth, viewportHeight);
   };
 
-  // update the camera information in the python server
-  const sendCamera = () => {
-    if (websocket.readyState === WebSocket.OPEN) {
-      const cmd = 'write';
-      const path = 'renderingState/camera';
-      const data_packet = sceneTree.metadata.camera.toJSON();
-      data_packet.object.timestamp = +new Date();
-      const data = {
-        type: cmd,
-        path,
-        data: data_packet,
-      };
-      const message = msgpack.encode(data);
-      websocket.send(message);
-    }
+  const render = () => {
+    const fps = 24;
+    const interval = 1000 / fps;
+    handleResize();
+    sceneTree.metadata.camera.updateProjectionMatrix();
+    sceneTree.metadata.moveCamera();
+    sceneTree.metadata.camera_controls.update(interval);
+    requestAnimationFrame(render);
+    renderer.render(scene, sceneTree.metadata.camera);
+    labelRenderer.render(scene, sceneTree.metadata.camera);
   };
 
   useEffect(() => {
@@ -180,35 +176,13 @@ export default function ViewerWindow(props) {
       height: get_window_height(),
       width: get_window_width(),
     });
+    render();
 
     return () => {
       window.removeEventListener('resize', update_dimensions);
     };
   }, []);
 
-  // keep sending the camera often
-  // rerun this when the websocket changes
-  useEffect(() => {
-    const fps = 24;
-    const interval = 1000 / fps;
-    const refreshIntervalId = setInterval(sendCamera, interval);
-    return () => {
-      clearInterval(refreshIntervalId);
-    };
-  }, [websocket]);
-
-  const render = () => {
-    const fps = 24;
-    const interval = 1000 / fps;
-    handleResize();
-    sceneTree.metadata.camera.updateProjectionMatrix();
-    sceneTree.metadata.moveCamera();
-    sceneTree.metadata.camera_controls.update(interval);
-    requestAnimationFrame(render);
-    renderer.render(scene, sceneTree.metadata.camera);
-    labelRenderer.render(scene, sceneTree.metadata.camera);
-  };
-
   // start the three.js rendering loop
   // when the DOM is ready
   useEffect(() => {
@@ -216,11 +190,6 @@ export default function ViewerWindow(props) {
     render();
   }, []);
 
-  // updates the current camera with the field of view
-  useEffect(() => {
-    sceneTree.metadata.camera.fov = field_of_view;
-  }, [field_of_view]);
-
   const render_height = useSelector(
     (state) => state.renderingState.render_height,
   );
@@ -265,8 +234,44 @@ export default function ViewerWindow(props) {
     const fl_new = fl * render_viewport_apsect_ratio;
     const fov = Math.atan(1 / fl_new) / (Math.PI / 360);
     sceneTree.metadata.camera.fov = fov;
+  } else {
+    sceneTree.metadata.camera.fov = 50;
   }
 
+  // update the camera information in the python server
+  const sendCamera = () => {
+    if (websocket.readyState === WebSocket.OPEN) {
+      const cmd = 'write';
+      const path = 'renderingState/camera';
+      const data_packet = sceneTree.metadata.camera.toJSON();
+      data_packet.object.timestamp = +new Date();
+      if (camera_choice === 'Render Camera') {
+        data_packet.object.camera_type = camera_type;
+      } else {
+        data_packet.object.camera_type = 'perspective';
+      }
+      data_packet.object.render_aspect = render_aspect;
+      const data = {
+        type: cmd,
+        path,
+        data: data_packet,
+      };
+      const message = msgpack.encode(data);
+      websocket.send(message);
+    }
+  };
+
+  // keep sending the camera often
+  // rerun this when the websocket changes
+  useEffect(() => {
+    const fps = 24;
+    const interval = 1000 / fps;
+    const refreshIntervalId = setInterval(sendCamera, interval);
+    return () => {
+      clearInterval(refreshIntervalId);
+    };
+  }, [websocket, camera_choice, camera_type, render_aspect]);
+
   return (
     <>
       {/* the webrtc viewer needs to know the camera pose */}

nerfstudio/viewer/app/src/reducer.js

@@ -22,6 +22,7 @@ const initialState = {
     render_height: 1080,
     render_width: 1920,
     field_of_view: 50,
+    camera_type: 'perspective',
 
     isTraining: true,
     output_options: ['rgb'], // populated by the possible Graph outputs

nerfstudio/viewer/server/utils.py

@@ -77,8 +77,19 @@ def get_intrinsics_matrix_and_camera_to_world_h(
     image_width = aspect * image_height
     pp_w = image_width / 2.0
     pp_h = image_height / 2.0
-    focal_length = three_js_perspective_camera_focal_length(fov, image_height)
-    intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float()
+    if (camera_object["camera_type"] == "perspective") | (camera_object["camera_type"] == "fisheye"):
+        focal_length = three_js_perspective_camera_focal_length(fov, image_height)
+        intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float()
+    elif camera_object["camera_type"] == "equirectangular":
+        render_aspect = camera_object["render_aspect"]
+        if aspect < render_aspect:
+            intrinsics_matrix = torch.tensor(
+                [[pp_w, 0, pp_w], [0, image_width / render_aspect, pp_h], [0, 0, 1]]
+            ).float()
+        else:
+            intrinsics_matrix = torch.tensor(
+                [[image_height * render_aspect / 2, 0, pp_w], [0, pp_h * 2, pp_h], [0, 0, 1]]
+            ).float()
 
     # extrinsics
     camera_to_world_h = torch.tensor(get_chunks(camera_object["matrix"], size_of_chunk=4)).T.float()

nerfstudio/viewer/server/viewer_utils.py

@@ -37,7 +37,7 @@ from aiortc import (
 from cryptography.utils import CryptographyDeprecationWarning
 from rich.console import Console
 
-from nerfstudio.cameras.cameras import Cameras
+from nerfstudio.cameras.cameras import Cameras, CameraType
 from nerfstudio.cameras.rays import RayBundle
 from nerfstudio.configs import base_config as cfg
 from nerfstudio.data.datasets.base_dataset import InputDataset
@@ -776,11 +776,22 @@ class ViewerState:
         if times is not None:
             times = torch.tensor([float(times)])
 
+        camera_type_msg = camera_object["camera_type"]
+        if camera_type_msg == "perspective":
+            camera_type = CameraType.PERSPECTIVE
+        elif camera_type_msg == "fisheye":
+            camera_type = CameraType.FISHEYE
+        elif camera_type_msg == "equirectangular":
+            camera_type = CameraType.EQUIRECTANGULAR
+        else:
+            camera_type = CameraType.PERSPECTIVE
+
         camera = Cameras(
             fx=intrinsics_matrix[0, 0],
             fy=intrinsics_matrix[1, 1],
             cx=intrinsics_matrix[0, 2],
             cy=intrinsics_matrix[1, 2],
+            camera_type=camera_type,
             camera_to_worlds=camera_to_world[None, ...],
             times=times,
         )