From c338de4aa4b0509e3468508c88d7e5981990edc1 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sun, 18 Dec 2022 10:46:39 -0800 Subject: [PATCH] Support non-pinhole rendering (#1141) --- nerfstudio/cameras/camera_paths.py | 30 ++++- nerfstudio/cameras/cameras.py | 2 +- nerfstudio/viewer/app/package.json | 2 +- .../SidePanel/CameraPanel/CameraPanel.jsx | 115 +++++++++++------- .../SidePanel/CameraPanel/CameraPropPanel.jsx | 22 ++++ .../src/modules/ViewerWindow/ViewerWindow.jsx | 91 +++++++------- nerfstudio/viewer/app/src/reducer.js | 1 + nerfstudio/viewer/server/utils.py | 15 ++- nerfstudio/viewer/server/viewer_utils.py | 13 +- 9 files changed, 191 insertions(+), 100 deletions(-) diff --git a/nerfstudio/cameras/camera_paths.py b/nerfstudio/cameras/camera_paths.py index fb243917d..f5eb67501 100644 --- a/nerfstudio/cameras/camera_paths.py +++ b/nerfstudio/cameras/camera_paths.py @@ -23,7 +23,7 @@ import torch import nerfstudio.utils.poses as pose_utils from nerfstudio.cameras import camera_utils from nerfstudio.cameras.camera_utils import get_interpolated_poses_many -from nerfstudio.cameras.cameras import Cameras +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.viewer.server.utils import three_js_perspective_camera_focal_length @@ -121,6 +121,19 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: image_height = camera_path["render_height"] image_width = camera_path["render_width"] + if "camera_type" in camera_path: + camera_type = camera_path["camera_type"] + else: + camera_type = "perspective" + + if "camera_type" not in camera_path: + camera_type = CameraType.PERSPECTIVE + elif camera_path["camera_type"] == "fisheye": + camera_type = CameraType.FISHEYE + elif camera_path["camera_type"] == "equirectangular": + camera_type = CameraType.EQUIRECTANGULAR + else: + camera_type = CameraType.PERSPECTIVE c2ws = [] fxs = [] @@ -129,11 +142,15 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: # pose c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3] c2ws.append(c2w) - # field of view - fov = camera["fov"] - focal_length = three_js_perspective_camera_focal_length(fov, image_height) - fxs.append(focal_length) - fys.append(focal_length) + if camera_type == CameraType.EQUIRECTANGULAR: + fxs.append(image_width / 2) + fys.append(image_height) + else: + # field of view + fov = camera["fov"] + focal_length = three_js_perspective_camera_focal_length(fov, image_height) + fxs.append(focal_length) + fys.append(focal_length) camera_to_worlds = torch.stack(c2ws, dim=0) fx = torch.tensor(fxs) @@ -144,4 +161,5 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: cx=image_width / 2, cy=image_height / 2, camera_to_worlds=camera_to_worlds, + camera_type=camera_type, ) diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 0299fa4bb..eb10f414b 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -634,7 +634,7 @@ class Cameras(TensorDataclass): directions_stack[..., 0][mask] = torch.masked_select(coord_stack[..., 0] * sin_theta / theta, mask).float() directions_stack[..., 1][mask] = torch.masked_select(coord_stack[..., 1] * sin_theta / theta, mask).float() - directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask) + directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask).float() if CameraType.EQUIRECTANGULAR.value in cam_types: mask = (self.camera_type[true_indices] == CameraType.EQUIRECTANGULAR.value).squeeze(-1) # (num_rays) diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index b6183729e..78d4997f3 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "22-12-02-0", + "version": "22-12-16-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 9958ea7b1..4efab1b6f 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -112,12 +112,12 @@ function FovSelector(props) { camera.setFocalLength(val / camera.aspect); } } else if (fovLabel === FOV_LABELS.FOV) { + camera.fov = val; setGlobalFov(val); } else { - const old_fov = camera.fov; camera.setFocalLength(val / camera.aspect); const new_fov = camera.getEffectiveFOV(); - camera.fov = old_fov; + camera.fov = new_fov; setGlobalFov(new_fov); } @@ -216,6 +216,8 @@ function CameraList(props) { const [slider_value, set_slider_value] = React.useState(0); const [expanded, setExpanded] = React.useState(null); + const camera_type = useSelector((state) => state.renderingState.camera_type); + const handleChange = (cameraUUID: string) => (event: React.SyntheticEvent, isExpanded: boolean) => { @@ -399,7 +401,7 @@ function CameraList(props) { - {isAnimated('FOV') && ( + {isAnimated('FOV') && camera_type !== 'equirectangular' && ( state.renderingState.render_width, ); + const camera_type = useSelector((state) => state.renderingState.camera_type); const setRenderHeight = (value) => { dispatch({ @@ -501,6 +504,22 @@ export default function CameraPanel(props) { }); }; + const setCameraType = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/camera_type', + data: value, + }); + }; + + const setFieldOfView = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/field_of_view', + data: value, + }); + }; + // ui state const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); @@ -516,7 +535,7 @@ export default function CameraPanel(props) { // set slider and render camera back to 0 if (new_camera_list.length >= 1) { set_camera_position(camera_render, new_camera_list[0].matrix); - camera_render.fov = new_camera_list[0].fov; + setFieldOfView(new_camera_list[0].fov); set_slider_value(slider_min); } }; @@ -718,7 +737,7 @@ export default function CameraPanel(props) { const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); - camera_render.fov = fov; + setFieldOfView(fov); } else { sceneTree.delete(['Camera Path', 'Curve']); } @@ -768,7 +787,7 @@ export default function CameraPanel(props) { fov = curve_object.curve_fovs.getPoint(point).z; const mat = get_transform_matrix(position, lookat, up); set_camera_position(camera_render, mat); - camera_render.fov = fov; + setFieldOfView(fov); setGlobalFov(fov); } }, [slider_value, render_height, render_width]); @@ -828,6 +847,7 @@ export default function CameraPanel(props) { // const myData const camera_path_object = { keyframes, + camera_type, render_height, render_width, camera_path, @@ -871,6 +891,7 @@ export default function CameraPanel(props) { setRenderHeight(camera_path_object.render_height); setRenderWidth(camera_path_object.render_width); + setCameraType(camera_path_object.camera_type); setFps(camera_path_object.fps); setSeconds(camera_path_object.seconds); @@ -1024,52 +1045,54 @@ export default function CameraPanel(props) { /> -
- - - - -
+ sx={{ + mt: 1, + }} + > + + + + + + )}