diff --git a/docs/developer_guides/viewer/index.md b/docs/developer_guides/viewer/index.md index eebc29d32..93204a203 100644 --- a/docs/developer_guides/viewer/index.md +++ b/docs/developer_guides/viewer/index.md @@ -8,6 +8,7 @@ The viewer is built on [Viser](https://github.com/brentyi/viser/tree/main/viser) :titlesonly: custom_gui +viewer_control local_viewer ``` diff --git a/docs/developer_guides/viewer/viewer_control.md b/docs/developer_guides/viewer/viewer_control.md new file mode 100644 index 000000000..e6a84d3d7 --- /dev/null +++ b/docs/developer_guides/viewer/viewer_control.md @@ -0,0 +1,82 @@ +# Python Viewer Control + +Similar to [`ViewerElements`](./custom_gui.md), Nerfstudio includes supports a Python interface to the viewer through which you can: + +* Set viewer camera pose and FOV +* Set viewer scene crop +* Retrieve the current viewer camera matrix +* Install listeners for click events inside the viewer window + +## Usage + +First, instantiate a `ViewerControl` object as a class variable inside a model file. +Just like `ViewerElements`, you can create an instance inside any class which inherits from `nn.Module` +and is contained within the `Pipeline` object (for example the `Model`) + +```python +from nerfstudio.viewer.server.viewer_elements import ViewerControl + +class MyModel(nn.Module): # Must inherit from nn.Module + def __init__(self): + # Must be a class variable + self.viewer_control = ViewerControl() # no arguments +``` +## Get Camera Matrix +To get the current camera intrinsics and extrinsics, use the `get_camera` function. This returns a `nerfstudio.cameras.cameras.Cameras` object. This object can be used to generate `RayBundles`, retrieve intrinsics and extrinsics, and more. + +```python +from nerfstudio.viewer.server.viewer_elements import ViewerControl, ViewerButton + +class MyModel(nn.Module): # Must inherit from nn.Module + def __init__(self): + ... + def button_cb(button): + # example of using the get_camera function, pass img width and height + # returns a Cameras object with 1 camera + camera = self.viewer_control.get_camera(100,100) + if camera is None: + # returns None when the viewer is not connected yet + return + # get the camera pose + camera_extrinsics_matrix = camera.camera_to_worlds[0,...] # 3x4 matrix + # generate image RayBundle + bundle = camera.generate_rays(camera_indices=0) + # Compute depth, move camera, or whatever you want + ... + self.viewer_button = ViewerButton(name="Dummy Button",cb_hook=button_cb) +``` + +## Set Camera Properties +You can set the viewer camera position and FOV from python. +To set position, you must define a new camera position as well as a 3D "look at" point which the camera aims towards. +```python +from nerfstudio.viewer.server.viewer_elements import ViewerControl,ViewerButton + +class MyModel(nn.Module): # Must inherit from nn.Module + def __init__(self): + ... + def aim_at_origin(button): + # instant=False means the camera smoothly animates + # instant=True means the camera jumps instantly to the pose + self.viewer_control.set_pose(position=(1,1,1),look_at=(0,0,0),instant=False) + self.viewer_button = ViewerButton(name="Dummy Button",cb_hook=button_cb) +``` + +## Double-click Callbacks +We forward *double* clicks inside the viewer to the ViewerControl object, which you can use to interact with the scene. To do this, register a callback using `register_click_cb()`. The click is defined to be a ray that starts at the camera origin and passes through the click point on the screen, in world coordinates. + +```python +from nerfstudio.viewer.server.viewer_elements import ViewerControl,ViewerClick + +class MyModel(nn.Module): # must inherit from nn.Module + def __init__(self): + # Must be a class variable + self.viewer_control = ViewerControl() # no arguments + def click_cb(click: ViewerClick): + print(f"Click at {click.origin} in direction {click.direction}") + self.viewer_control.register_click_cb(click_cb) +``` + +### Thread safety +Just like `ViewerElement` callbacks, click callbacks are asynchronous to training and can potentially interrupt a call to `get_outputs()`. + diff --git a/nerfstudio/scripts/viewer/sync_viser_message_defs.py b/nerfstudio/scripts/viewer/sync_viser_message_defs.py index 2e4e0ffc8..5053a82ea 100644 --- a/nerfstudio/scripts/viewer/sync_viser_message_defs.py +++ b/nerfstudio/scripts/viewer/sync_viser_message_defs.py @@ -35,7 +35,7 @@ def main() -> None: # - nerfstudio/scripts/ # - nerfstudio/ target_path = pathlib.Path(__file__).absolute().parent.parent.parent / pathlib.Path( - "nerfstudio/viewer/app/src/modules/WebSocket/ViserMessages.tsx" + "viewer/app/src/modules/WebSocket/ViserMessages.tsx" ) assert target_path.exists() diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index d76e6e099..136739355 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-05-15-0", + "version": "23-05-15-1", "private": true, "dependencies": { "@emotion/react": "^11.10.4", @@ -16,7 +16,7 @@ "@testing-library/react": "^13.3.0", "@testing-library/user-event": "^14.2.0", "await-lock": "^2.2.2", - "camera-controls": "^1.37.2", + "camera-controls": "^2.0", "classnames": "^2.3.1", "dat.gui": "^0.7.9", "dayjs": "^1.11.5", @@ -93,4 +93,4 @@ "resolve-url-loader": "^5.0.0", "yaml": "^2.2.2" } -} +} \ No newline at end of file diff --git a/nerfstudio/viewer/app/src/App.jsx b/nerfstudio/viewer/app/src/App.jsx index fd42a37f9..1330c1fcb 100644 --- a/nerfstudio/viewer/app/src/App.jsx +++ b/nerfstudio/viewer/app/src/App.jsx @@ -6,6 +6,7 @@ import Banner from './modules/Banner'; import { BasicTabs } from './modules/SidePanel/SidePanel'; import ViewerWindow from './modules/ViewerWindow/ViewerWindow'; import { appTheme } from './themes/theme'; +import { ViserWebSocket } from './modules/WebSocket/ViserWebSocket'; export default function App() { // The scene tree won't rerender but it will listen to changes @@ -14,19 +15,21 @@ export default function App() { const sceneTree = get_scene_tree(); return ( - - -
- {/* The banner at the top of the page. */} - -
- {/* Order matters here. The viewer window must be rendered first. */} - -
- + + + +
+ {/* The banner at the top of the page. */} + +
+ {/* Order matters here. The viewer window must be rendered first. */} + +
+ +
-
- + + ); } diff --git a/nerfstudio/viewer/app/src/index.jsx b/nerfstudio/viewer/app/src/index.jsx index b712d1d3d..95b6068df 100644 --- a/nerfstudio/viewer/app/src/index.jsx +++ b/nerfstudio/viewer/app/src/index.jsx @@ -4,13 +4,10 @@ import { createRoot } from 'react-dom/client'; import { Provider } from 'react-redux'; import App from './App'; import store from './store'; -import { ViserWebSocket } from './modules/WebSocket/ViserWebSocket'; const root = createRoot(document.getElementById('root')); root.render( - - , ); diff --git a/nerfstudio/viewer/app/src/index.scss b/nerfstudio/viewer/app/src/index.scss index 83e09b372..3a63228b6 100644 --- a/nerfstudio/viewer/app/src/index.scss +++ b/nerfstudio/viewer/app/src/index.scss @@ -30,12 +30,14 @@ $highlight-color1: #606980; $highlight-color2: #eeeeee; $highlight-color3: #ffffff; +$bannerHeight: 50; + .banner { position: absolute; top: 0; left: 0; right: 0; - height: 50px; + height: $bannerHeight + 0px; width: 100%; z-index: 1000; background-color: $base-color2; @@ -52,6 +54,10 @@ $highlight-color3: #ffffff; } } +:export { + bannerHeight: $bannerHeight; +} + .App-body { position: absolute; top: 50px; diff --git a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx index 3137e2405..8d9e074af 100644 --- a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx @@ -39,11 +39,11 @@ function CustomLeva() { guiNames.forEach((key) => { const { levaConf, folderLabels, hidden } = guiConfigFromName[key]; - if (hidden) return; const leafFolder = getFolderContainer(folderLabels); // Hacky stuff that lives outside of TypeScript... if (levaConf.type === 'BUTTON') { + if (hidden) return; // Add a button. leafFolder[key] = button(() => { sendWebsocketMessage(viser_websocket, { @@ -79,6 +79,7 @@ function CustomLeva() { value, }); }, + render: () => !hidden, }; } }); diff --git a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx index 197f6887f..d2dbc15de 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx +++ b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx @@ -6,13 +6,15 @@ import { CSS2DRenderer } from 'three/examples/jsm/renderers/CSS2DRenderer'; import { TransformControls } from 'three/examples/jsm/controls/TransformControls'; import { useDispatch } from 'react-redux'; -import { drawCamera, drawSceneBox } from './drawing'; +import { drawCamera, drawSceneBox} from './drawing'; import { CameraHelper } from '../SidePanel/CameraPanel/CameraHelper'; import SceneNode from '../../SceneNode'; import { subscribe_to_changes } from '../../subscriber'; import { snap_to_camera } from '../SidePanel/SidePanel'; +import variables from '../../index.scss'; + const SCENE_BOX_NAME = 'Scene Box'; const CAMERAS_NAME = 'Training Cameras'; @@ -39,7 +41,7 @@ export function get_scene_tree() { // eslint-disable-next-line react-hooks/rules-of-hooks const dispatch = useDispatch(); - const BANNER_HEIGHT = 50; + const BANNER_HEIGHT = parseInt(variables.bannerHeight,10); // Main camera const main_camera = new THREE.PerspectiveCamera(75, 1, 0.1, 1000); @@ -83,16 +85,21 @@ export function get_scene_tree() { CameraControls.install({ THREE }); const camera_controls = new CameraControls(main_camera, renderer.domElement); - camera_controls.azimuthRotateSpeed = 0.3; - camera_controls.polarRotateSpeed = 0.3; + camera_controls.azimuthRotateSpeed = 0.4; + camera_controls.polarRotateSpeed = 0.4; camera_controls.dollySpeed = 0.1; camera_controls.infinityDolly = true; + camera_controls.smoothTime=.05; + camera_controls.draggingSmoothTime=.05; + camera_controls.restThreshold = .0025; camera_controls.saveState(); const keyMap = []; - const moveSpeed = 0.008; - const rotSpeed = 0.015; - const EPS = 0.01; + const moveSpeed = 0.005; + const upRotSpeed = 0.04; + const sideRotSpeed = .01; + const EPS = 0.005; + function rotate() { if ( @@ -108,29 +115,30 @@ export function get_scene_tree() { curPos.x + diff.x, curPos.y + diff.y, curPos.z + diff.z, + {enableTransition:true} ); if (keyMap.ArrowLeft === true) { - camera_controls.rotate(rotSpeed, 0, true); + camera_controls.rotate(sideRotSpeed, 0, {enableTransition:true}); } if (keyMap.ArrowRight === true) { - camera_controls.rotate(-rotSpeed, 0, true); + camera_controls.rotate(-sideRotSpeed, 0, {enableTransition:true}); } if (keyMap.ArrowUp === true) { - camera_controls.rotate(0, rotSpeed / 1.5, true); + camera_controls.rotate(0, upRotSpeed, {enableTransition:true}); } if (keyMap.ArrowDown === true) { - camera_controls.rotate(0, -rotSpeed / 1.5, true); + camera_controls.rotate(0, -upRotSpeed, {enableTransition:true}); } } } function translate() { if (keyMap.KeyD === true) { - camera_controls.truck(moveSpeed, 0, true); + camera_controls.truck(moveSpeed, 0, {enableTransition:true}); } if (keyMap.KeyA === true) { - camera_controls.truck(-moveSpeed, 0, true); + camera_controls.truck(-moveSpeed, 0, {enableTransition:true}); } if (keyMap.KeyW === true) { const curPos = camera_controls.getPosition(); @@ -143,17 +151,18 @@ export function get_scene_tree() { curPos.x + newDiff.x, curPos.y + newDiff.y, curPos.z + newDiff.z, + {enableTransition:true} ); - camera_controls.dolly(moveSpeed, true); + camera_controls.dolly(moveSpeed, {enableTransition:true}); } if (keyMap.KeyS === true) { - camera_controls.dolly(-moveSpeed, true); + camera_controls.dolly(-moveSpeed, {enableTransition:true}); } if (keyMap.KeyQ === true) { - camera_controls.truck(0, moveSpeed, true); + camera_controls.truck(0, moveSpeed, {enableTransition:true}); } if (keyMap.KeyE === true) { - camera_controls.truck(0, -moveSpeed, true); + camera_controls.truck(0, -moveSpeed, {enableTransition:true}); } } @@ -162,7 +171,7 @@ export function get_scene_tree() { return; } if (keyMap.Space === true) { - camera_controls.setLookAt(0.7, -0.7, 0.3, 0, 0, 0); + camera_controls.setLookAt(0.7, -0.7, 0.3, 0, 0, 0, {enableTransition:true}); } translate(); rotate(); @@ -302,6 +311,7 @@ export function get_scene_tree() { curPos.x + newDiff.x, curPos.y + newDiff.y, curPos.z + newDiff.z, + {enableTransition:true} ); }; diff --git a/nerfstudio/viewer/app/src/modules/Scene/drawing.js b/nerfstudio/viewer/app/src/modules/Scene/drawing.js index 9de449a5e..e64f3214b 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/drawing.js +++ b/nerfstudio/viewer/app/src/modules/Scene/drawing.js @@ -162,4 +162,4 @@ export function drawCameras(cameras): Record { cameraObjects[key] = drawCamera(camera); } return cameraObjects; -} +} \ No newline at end of file diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/ScenePanel/ScenePanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/ScenePanel/ScenePanel.jsx index 254d75008..e255249f8 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/ScenePanel/ScenePanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/ScenePanel/ScenePanel.jsx @@ -32,6 +32,7 @@ export const snap_to_camera = (sceneTree, camera, matrix) => { viewDirection.x, viewDirection.y, viewDirection.z, + {enableTransition:true} ); }; diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx index 1e7db37e3..9526bfdaf 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx @@ -33,6 +33,7 @@ export const snap_to_camera = (sceneTree, camera, matrix) => { viewDirection.x, viewDirection.y, viewDirection.z, + {enableTransition:true} ); }; diff --git a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx index 7415f09a0..a8a596c33 100644 --- a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx +++ b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx @@ -17,6 +17,8 @@ import { ViserWebSocketContext, } from '../WebSocket/ViserWebSocket'; +import variables from '../../index.scss'; + function CameraToggle() { const dispatch = useDispatch(); const camera_choice = useSelector( @@ -149,14 +151,14 @@ export default function ViewerWindow(props) { renderer.setSize(viewportWidth, viewportHeight); labelRenderer.setSize(viewportWidth, viewportHeight); }; + const clock = new THREE.Clock(); const render = () => { - const fps = 24; - const interval = 1000 / fps; + const delta = clock.getDelta(); handleResize(); sceneTree.metadata.camera.updateProjectionMatrix(); sceneTree.metadata.moveCamera(); - sceneTree.metadata.camera_controls.update(interval); + sceneTree.metadata.camera_controls.update(delta); requestAnimationFrame(render); renderer.render(scene, sceneTree.metadata.camera); labelRenderer.render(scene, sceneTree.metadata.camera); @@ -191,7 +193,7 @@ export default function ViewerWindow(props) { const oldBackground = scene.background; const texture = new THREE.Texture(this); texture.minFilter = THREE.LinearFilter; - texture.magFilter = THREE.NearestFilter; + texture.magFilter = THREE.LinearFilter; texture.needsUpdate = true; scene.background = texture; if (oldBackground) { @@ -313,6 +315,49 @@ export default function ViewerWindow(props) { // eslint-disable-next-line react-hooks/exhaustive-deps }, [isWebsocketConnected]); + const throttledClickSender = makeThrottledMessageSender( + viser_websocket, + 10, + ); + useEffect(() => { + const onMouseDouble = (e) => { + const BANNER_HEIGHT = parseInt(variables.bannerHeight,10); + + const mouseVector = new THREE.Vector2(); + mouseVector.x = 2 * (e.clientX / size.x) - 1; + mouseVector.y = 1 - 2 * ((e.clientY - BANNER_HEIGHT) / size.y); + + const mouse_in_scene = !( + mouseVector.x > 1 || + mouseVector.x < -1 || + mouseVector.y > 1 || + mouseVector.y < -1 + ); + if (!mouse_in_scene) { return; } + + const raycaster = new THREE.Raycaster(); + raycaster.setFromCamera(mouseVector, sceneTree.metadata.camera); + + throttledClickSender({ + type: 'ClickMessage', + origin: [ + raycaster.ray.origin.x, + raycaster.ray.origin.y, + raycaster.ray.origin.z + ], + direction: [ + raycaster.ray.direction.x, + raycaster.ray.direction.y, + raycaster.ray.direction.z + ], + }); + }; + window.addEventListener('dblclick', onMouseDouble, false); + return () => { + window.removeEventListener('dblclick', onMouseDouble, false); + }; + }, [size, sceneTree.metadata.camera, throttledClickSender]); + return ( <>
diff --git a/nerfstudio/viewer/app/src/modules/WebSocket/ViserMessages.tsx b/nerfstudio/viewer/app/src/modules/WebSocket/ViserMessages.tsx index 965e59ab6..405442e86 100644 --- a/nerfstudio/viewer/app/src/modules/WebSocket/ViserMessages.tsx +++ b/nerfstudio/viewer/app/src/modules/WebSocket/ViserMessages.tsx @@ -41,6 +41,13 @@ interface FilePathInfoMessage { data_base_dir: string; export_path_name: string; } +interface SetCameraMessage { + type: "SetCameraMessage"; + fov: number | null; + look_at: [number, number, number] | null; + position: [number, number, number] | null; + instant: boolean; +} interface CameraMessage { type: "CameraMessage"; aspect: number; @@ -99,6 +106,11 @@ interface TimeConditionMessage { type: "TimeConditionMessage"; time: number; } +interface ClickMessage { + type: "ClickMessage"; + origin: [number, number, number]; + direction: [number, number, number]; +} interface OutputOptionsMessage { type: "OutputOptionsMessage"; options: any; @@ -113,6 +125,7 @@ export type Message = | GuiSetValueMessage | GuiSetLevaConfMessage | FilePathInfoMessage + | SetCameraMessage | CameraMessage | SceneBoxMessage | DatasetImageMessage @@ -125,4 +138,5 @@ export type Message = | SaveCheckpointMessage | UseTimeConditioningMessage | TimeConditionMessage + | ClickMessage | OutputOptionsMessage; diff --git a/nerfstudio/viewer/app/src/modules/WebSocket/ViserWebSocket.tsx b/nerfstudio/viewer/app/src/modules/WebSocket/ViserWebSocket.tsx index 0b5b8f2e9..57d33f541 100644 --- a/nerfstudio/viewer/app/src/modules/WebSocket/ViserWebSocket.tsx +++ b/nerfstudio/viewer/app/src/modules/WebSocket/ViserWebSocket.tsx @@ -1,10 +1,12 @@ import React, { useEffect, MutableRefObject, Dispatch } from 'react'; - import AwaitLock from 'await-lock'; import { pack, unpack } from 'msgpackr'; import { useDispatch, useStore, useSelector } from 'react-redux'; import { Store } from 'redux'; +import {MathUtils} from 'three'; import { Message } from './ViserMessages'; +import SceneNode from '../../SceneNode'; + const ViserWebSocketContext = React.createContext | null>(null); @@ -53,6 +55,7 @@ function handleMessage( message: Message, dispatch: Dispatch, store: Store, + sceneTree: SceneNode, ) { switch (message.type) { // Add a background image. @@ -159,6 +162,36 @@ function handleMessage( } break; } + // Set camera position + case 'SetCameraMessage': { + if(message.fov !== null){ + const cam = sceneTree.metadata.camera; + const vExtentSlope = Math.tan( MathUtils.DEG2RAD * 0.5 * message.fov ); + + const newFocalLength = 0.5 * cam.getFilmHeight() / vExtentSlope; + cam.setFocalLength(newFocalLength); + } + if(message.look_at !==null && message.position !== null){ + sceneTree.metadata.camera_controls.setLookAt(message.position[0], + message.position[1], + message.position[2], + message.look_at[0], + message.look_at[1], + message.look_at[2], + !message.instant); + } + else{ + if(message.look_at !==null){ + const p = message.look_at; + sceneTree.metadata.camera_controls.setTarget(p[0],p[1],p[2],!message.instant); + } + if(message.position !==null){ + const p = message.position; + sceneTree.metadata.camera_controls.setPosition(p[0],p[1],p[2],!message.instant); + } + } + break; + } // Remove a GUI input. case 'GuiRemoveMessage': { // TODO: not implemented. @@ -291,7 +324,7 @@ function handleMessage( } } -export function ViserWebSocket({ children }: { children: React.ReactNode }) { +export function ViserWebSocket({sceneTree, children}: {sceneTree: SceneNode, children: React.ReactNode}) { const dispatch = useDispatch(); const store = useStore(); @@ -352,7 +385,7 @@ export function ViserWebSocket({ children }: { children: React.ReactNode }) { new Uint8Array(await event.data.arrayBuffer()), )) as Message; await orderLock.acquireAsync({ timeout: 1000 }); - handleMessage(message, dispatch, store); + handleMessage(message, dispatch, store, sceneTree); } catch (error) { console.error(`Error handling message: ${error}`); } finally { diff --git a/nerfstudio/viewer/app/yarn.lock b/nerfstudio/viewer/app/yarn.lock index 24d156e1f..dada0632a 100644 --- a/nerfstudio/viewer/app/yarn.lock +++ b/nerfstudio/viewer/app/yarn.lock @@ -3599,10 +3599,10 @@ camelcase@^6.2.0, camelcase@^6.2.1: resolved "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -camera-controls@^1.37.2: - version "1.37.2" - resolved "https://registry.npmjs.org/camera-controls/-/camera-controls-1.37.2.tgz" - integrity sha512-b+8DYJc/honm3b/PzjTQbVjsnkdgiDnI44/PRAWKlLhF0lzggGtaU9ZSGRM/lll2HkF0KN+OHx/ZN80wJmEbqA== +camera-controls@^2.0: + version "2.3.4" + resolved "https://registry.yarnpkg.com/camera-controls/-/camera-controls-2.3.4.tgz#f372b5a4012b474df0ed0c67da7a3198d7efee0b" + integrity sha512-swhc87YVHf9te0glBI7Oa/QBgsSCL4ZxtoR4V3vE6l7mEebsYRNL8y7Y2m2E6MrT0UTphM1S7mQqs0Sp7QTZ2g== caniuse-api@^3.0.0: version "3.0.0" diff --git a/nerfstudio/viewer/server/render_state_machine.py b/nerfstudio/viewer/server/render_state_machine.py index f49cba2b6..1ad691b28 100644 --- a/nerfstudio/viewer/server/render_state_machine.py +++ b/nerfstudio/viewer/server/render_state_machine.py @@ -22,12 +22,11 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple, get_args import torch -from nerfstudio.cameras.cameras import Cameras, CameraType +from nerfstudio.cameras.cameras import Cameras from nerfstudio.model_components.renderers import background_color_override_context from nerfstudio.utils import colormaps, writer from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfstudio.viewer.server import viewer_utils -from nerfstudio.viewer.server.utils import get_intrinsics_matrix_and_camera_to_world_h from nerfstudio.viewer.viser.messages import CameraMessage if TYPE_CHECKING: @@ -124,40 +123,8 @@ class RenderStateMachine(threading.Thread): image_height, image_width = self._calculate_image_res(cam_msg.aspect) - intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h( - cam_msg, image_height=image_height, image_width=image_width - ) - - camera_to_world = camera_to_world_h[:3, :] - camera_to_world = torch.stack( - [ - camera_to_world[0, :], - camera_to_world[2, :], - camera_to_world[1, :], - ], - dim=0, - ) - - camera_type_msg = cam_msg.camera_type - if camera_type_msg == "perspective": - camera_type = CameraType.PERSPECTIVE - elif camera_type_msg == "fisheye": - camera_type = CameraType.FISHEYE - elif camera_type_msg == "equirectangular": - camera_type = CameraType.EQUIRECTANGULAR - else: - camera_type = CameraType.PERSPECTIVE - - camera = Cameras( - fx=intrinsics_matrix[0, 0], - fy=intrinsics_matrix[1, 1], - cx=intrinsics_matrix[0, 2], - cy=intrinsics_matrix[1, 2], - camera_type=camera_type, - camera_to_worlds=camera_to_world[None, ...], - times=torch.tensor([self.viewer.control_panel.time], dtype=torch.float32), - ) - camera = camera.to(self.viewer.get_model().device) + camera: Optional[Cameras] = self.viewer.get_camera(image_height, image_width) + assert camera is not None, "render called before viewer connected" with self.viewer.train_lock if self.viewer.train_lock is not None else contextlib.nullcontext(): camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=self.viewer.get_model().render_aabb) diff --git a/nerfstudio/viewer/server/viewer_elements.py b/nerfstudio/viewer/server/viewer_elements.py index 425cf3737..802e0d2eb 100644 --- a/nerfstudio/viewer/server/viewer_elements.py +++ b/nerfstudio/viewer/server/viewer_elements.py @@ -15,18 +15,135 @@ """ Viewer GUI elements for the nerfstudio viewer """ + from __future__ import annotations from abc import abstractmethod -from typing import Any, Callable, Generic, List, Optional, Tuple +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Callable, Generic, List, Optional, Tuple from typing_extensions import TypeVar +from nerfstudio.cameras.cameras import Cameras from nerfstudio.viewer.viser import GuiHandle, GuiSelectHandle, ViserServer +from nerfstudio.viewer.viser.messages import ClickMessage + +if TYPE_CHECKING: + from nerfstudio.viewer.server.control_panel import ControlPanel + from nerfstudio.viewer.server.viewer_state import ViewerState + TValue = TypeVar("TValue") +@dataclass +class ViewerClick: + """ + Class representing a click in the viewer as a ray. + """ + + # the information here matches the information in the ClickMessage, + # but we implement a wrapper as an abstraction layer + origin: Tuple[float, float, float] + """The origin of the click in world coordinates (center of camera)""" + direction: Tuple[float, float, float] + """ + The direction of the click if projected from the camera through the clicked pixel, + in world coordinates + """ + + +class ViewerControl: + """ + class for exposing non-gui controls of the viewer to the user + """ + + def __init__(self): + # this should be a user-facing constructor, since it will be used inside the model/pipeline class + self.click_cbs = [] + + def _setup(self, viewer_state: ViewerState): + """ + Internal use only, setup the viewer control with the viewer state object + + Args: + viewer_state: The viewer state object (viewer_state.py) + """ + self.viewer_state = viewer_state + self.control_panel: ControlPanel = viewer_state.control_panel + self.viser_server: ViserServer = viewer_state.viser_server + + def set_pose( + self, + position: Optional[Tuple[float, float, float]] = None, + look_at: Optional[Tuple[float, float, float]] = None, + instant: bool = False, + ): + """ + Set the camera position of the viewer camera. + + Args: + position: The new position of the camera in world coordinates + look_at: The new look_at point of the camera in world coordinates + instant: If the camera should move instantly or animate to the new position + """ + assert hasattr(self, "viser_server"), "Called set_position on uninitialized ViewerControl" + self.viser_server.set_camera(position=position, look_at=look_at, instant=instant) + + def set_fov(self, fov): + """ + Set the FOV of the viewer camera + + Args: + fov: The new FOV of the camera in degrees + + """ + assert hasattr(self, "viser_server"), "Called set_fov on uninitialized ViewerControl" + self.viser_server.set_camera(fov=fov) + + def set_crop(self, min_point: Tuple[float, float, float], max_point: Tuple[float, float, float]): + """ + Set the scene crop box of the viewer to the specified min,max point + + Args: + min_point: The minimum point of the crop box + max_point: The maximum point of the crop box + + """ + assert hasattr(self, "viser_server"), "Called set_crop on uninitialized ViewerControl" + self.control_panel.crop_min = min_point + self.control_panel.crop_max = max_point + + def get_camera(self, img_height: int, img_width: int) -> Optional[Cameras]: + """ + Returns the Cameras object representing the current camera for the viewer, or None if the viewer + is not connected yet + + Args: + img_height: The height of the image to get camera intrinsics for + img_width: The width of the image to get camera intrinsics for + """ + return self.viewer_state.get_camera(img_height, img_width) + + def register_click_cb(self, cb: Callable): + """ + Add a callback which will be called when a click is detected in the viewer. + + Args: + cb: The callback to call when a click is detected. + The callback should take a ViewerClick object as an argument + """ + self.click_cbs.append(cb) + + def on_click(self, msg: ClickMessage): + """ + Internal use only, register a click in the viewer which propagates to all self.click_cbs + """ + click = ViewerClick(origin=msg.origin, direction=msg.direction) + for c in self.click_cbs: + c(click) + + class ViewerElement(Generic[TValue]): """Base class for all viewer elements diff --git a/nerfstudio/viewer/server/viewer_state.py b/nerfstudio/viewer/server/viewer_state.py index 7c45d2152..838d26034 100644 --- a/nerfstudio/viewer/server/viewer_state.py +++ b/nerfstudio/viewer/server/viewer_state.py @@ -25,6 +25,7 @@ from rich import box, style from rich.panel import Panel from rich.table import Table +from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.configs import base_config as cfg from nerfstudio.data.datasets.base_dataset import InputDataset from nerfstudio.data.scene_box import SceneBox @@ -41,12 +42,14 @@ from nerfstudio.viewer.server.render_state_machine import ( RenderAction, RenderStateMachine, ) -from nerfstudio.viewer.server.viewer_elements import ViewerElement +from nerfstudio.viewer.server.utils import get_intrinsics_matrix_and_camera_to_world_h +from nerfstudio.viewer.server.viewer_elements import ViewerControl, ViewerElement from nerfstudio.viewer.viser import ViserServer from nerfstudio.viewer.viser.messages import ( CameraMessage, CameraPathOptionsRequest, CameraPathPayloadMessage, + ClickMessage, CropParamsMessage, NerfstudioMessage, SaveCheckpointMessage, @@ -128,6 +131,7 @@ class ViewerState: self.viser_server.register_handler(CameraPathOptionsRequest, self._handle_camera_path_option_request) self.viser_server.register_handler(CameraPathPayloadMessage, self._handle_camera_path_payload) self.viser_server.register_handler(CropParamsMessage, self._handle_crop_params_message) + self.viser_server.register_handler(ClickMessage, self._handle_click_message) if self.include_time: self.viser_server.use_time_conditioning() self.viser_server.register_handler(TimeConditionMessage, self._handle_time_condition_message) @@ -146,7 +150,7 @@ class ViewerState: element.install(self.viser_server) # also rewire the hook to rerender prev_cb = element.cb_hook - element.cb_hook = lambda element: [self._interrupt_render(element), prev_cb(element)] + element.cb_hook = lambda element: [prev_cb(element), self._interrupt_render(element)] else: with self.viser_server.gui_folder(folder_labels[0]): nested_folder_install(folder_labels[1:], element) @@ -160,6 +164,18 @@ class ViewerState: folder_labels = param_path.split("/")[:-1] nested_folder_install(folder_labels, element) + # scrape the trainer/pipeline for any ViewerControl objects to initialize them + if self.trainer is not None: + self.viewer_controls: List[ViewerControl] = [ + e for (_, e) in parse_object(self.trainer, ViewerControl, "Trainer") + ] + else: + self.viewer_controls: List[ViewerControl] = [ + e for (_, e) in parse_object(self.trainer, ViewerControl, "Pipeline") + ] + + for c in self.viewer_controls: + c._setup(self) self.render_statemachine = RenderStateMachine(self) self.render_statemachine.start() @@ -176,8 +192,6 @@ class ViewerState: def _crop_params_update(self, _) -> None: """Update crop parameters""" - if self.camera_message is not None: - self.render_statemachine.action(RenderAction("rerender", self.camera_message)) crop_min = torch.tensor(self.control_panel.crop_min, dtype=torch.float32) crop_max = torch.tensor(self.control_panel.crop_max, dtype=torch.float32) scene_box = SceneBox(aabb=torch.stack([crop_min, crop_max], dim=0)) @@ -190,6 +204,8 @@ class ViewerState: crop_scale=tuple(crop_scale.tolist()), crop_center=tuple(crop_center.tolist()), ) + if self.camera_message is not None: + self.render_statemachine.action(RenderAction("rerender", self.camera_message)) def _handle_training_state_message(self, message: NerfstudioMessage) -> None: """Handle training state message from viewer.""" @@ -248,6 +264,12 @@ class ViewerState: self.control_panel.crop_min = tuple(crop_min.tolist()) self.control_panel.crop_max = tuple(crop_max.tolist()) + def _handle_click_message(self, message: NerfstudioMessage) -> None: + """Handle click message from viewer.""" + assert isinstance(message, ClickMessage) + for controls in self.viewer_controls: + controls.on_click(message) + def _handle_time_condition_message(self, message: NerfstudioMessage) -> None: """Handle time conditioning message from viewer.""" assert isinstance(message, TimeConditionMessage) @@ -266,6 +288,49 @@ class ViewerState: if self.trainer is not None: self.trainer.training_state = training_state + def get_camera(self, image_height: int, image_width: int) -> Optional[Cameras]: + """ + Return a Cameras object representing the camera for the viewer given the provided image height and width + """ + cam_msg: Optional[CameraMessage] = self.camera_message + if cam_msg is None: + return None + intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h( + cam_msg, image_height=image_height, image_width=image_width + ) + + camera_to_world = camera_to_world_h[:3, :] + camera_to_world = torch.stack( + [ + camera_to_world[0, :], + camera_to_world[2, :], + camera_to_world[1, :], + ], + dim=0, + ) + + camera_type_msg = cam_msg.camera_type + if camera_type_msg == "perspective": + camera_type = CameraType.PERSPECTIVE + elif camera_type_msg == "fisheye": + camera_type = CameraType.FISHEYE + elif camera_type_msg == "equirectangular": + camera_type = CameraType.EQUIRECTANGULAR + else: + camera_type = CameraType.PERSPECTIVE + + camera = Cameras( + fx=intrinsics_matrix[0, 0], + fy=intrinsics_matrix[1, 1], + cx=intrinsics_matrix[0, 2], + cy=intrinsics_matrix[1, 2], + camera_type=camera_type, + camera_to_worlds=camera_to_world[None, ...], + times=torch.tensor([self.control_panel.time], dtype=torch.float32), + ) + camera = camera.to(self.get_model().device) + return camera + def _pick_drawn_image_idxs(self, total_num: int) -> list[int]: """Determine indicies of images to display in viewer. diff --git a/nerfstudio/viewer/viser/message_api.py b/nerfstudio/viewer/viser/message_api.py index 49df4e292..ec602e4df 100644 --- a/nerfstudio/viewer/viser/message_api.py +++ b/nerfstudio/viewer/viser/message_api.py @@ -523,6 +523,25 @@ class MessageApi(abc.ABC): """ self._queue(messages.TrainingStateMessage(training_state=training_state)) + def set_camera( + self, + position: Optional[Tuple[float, float, float]] = None, + look_at: Optional[Tuple[float, float, float]] = None, + fov: Optional[int] = None, + instant: bool = False, + ) -> None: + """Update the camera object in the viewer. If any of the arguments are None, the corresponding value will not + be set in the viewer. For example, setting position only will maintain the same look-at point while moving + the origin of the camera + + Args: + position: The position in world coordinates of the camera + look_at: The position in world coordinates of the new look at point + fov: The new field of view + instant: Whether to move the camera instantly or animate + """ + self._queue(messages.SetCameraMessage(look_at=look_at, position=position, fov=fov, instant=instant)) + def send_camera_paths(self, camera_paths: Dict[str, Any]) -> None: """Send camera paths to the scene. diff --git a/nerfstudio/viewer/viser/messages.py b/nerfstudio/viewer/viser/messages.py index dd9afb30d..2ce9de61d 100644 --- a/nerfstudio/viewer/viser/messages.py +++ b/nerfstudio/viewer/viser/messages.py @@ -18,7 +18,7 @@ from __future__ import annotations import dataclasses -from typing import Any, Literal, Tuple +from typing import Any, Literal, Optional, Tuple import viser.infra from typing_extensions import override @@ -124,6 +124,20 @@ class FilePathInfoMessage(NerfstudioMessage): """ Name of the export folder """ +@dataclasses.dataclass +class SetCameraMessage(NerfstudioMessage): + """Set the current camera.""" + + fov: Optional[float] + """ Field of view of the camera """ + look_at: Optional[Tuple[float, float, float]] + """Point in 3D the camera is looking at""" + position: Optional[Tuple[float, float, float]] + """ Position of the camera""" + instant: bool = False + """ Whether to move the camera instantly or animate it""" + + @dataclasses.dataclass class CameraMessage(NerfstudioMessage): """Render camera data.""" @@ -243,6 +257,16 @@ class TimeConditionMessage(NerfstudioMessage): """ Time conditioning value """ +@dataclasses.dataclass +class ClickMessage(NerfstudioMessage): + """Click message.""" + + origin: Tuple[float, float, float] + """The origin of the click in world coords (center of camera)""" + direction: Tuple[float, float, float] + """The direction of the click if projected through the clicked pixel (world coords)""" + + @dataclasses.dataclass class OutputOptionsMessage(NerfstudioMessage): """Output options message which are used in the export panel.