From 345ef84be6f709fae712c2e6b2600463cdb05aae Mon Sep 17 00:00:00 2001 From: Abhik Ahuja Date: Wed, 12 Oct 2022 20:42:22 -0700 Subject: [PATCH] Snap to cameras (#550) Clicking on training cameras now snaps to them Co-authored-by: mcallisterdavid --- .../viewer/app/src/modules/Scene/Scene.jsx | 62 ++++++++++++++++++- .../viewer/app/src/modules/Scene/drawing.js | 6 +- .../app/src/modules/SidePanel/SidePanel.jsx | 53 +++++++++++++++- 3 files changed, 116 insertions(+), 5 deletions(-) diff --git a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx index c968f3949..358e3ad58 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx +++ b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx @@ -13,6 +13,7 @@ import { CameraHelper } from '../SidePanel/CameraPanel/CameraHelper'; import SceneNode from '../../SceneNode'; import { WebSocketContext } from '../WebSocket/WebSocket'; import { subscribe_to_changes } from '../../subscriber'; +import { snap_to_camera } from '../SidePanel/SidePanel'; const msgpack = require('msgpack-lite'); @@ -203,7 +204,7 @@ export function get_scene_tree() { if (!prev.has(key)) { // keys_valid.push(key); const json = current[key]; - const camera = drawCamera(json); + const camera = drawCamera(json, key); sceneTree.set_object_from_path([CAMERAS_NAME, key], camera); } } @@ -220,6 +221,65 @@ export function get_scene_tree() { }; subscribe_to_changes(selector_fn_cameras, fn_value_cameras); + // Check for clicks on training cameras + const mouseVector = new THREE.Vector2(); + const raycaster = new THREE.Raycaster(); + const size = new THREE.Vector2(); + let selectedCam = null; + + let drag = false; + const onMouseDown = () => { + drag = false; + }; + + const onMouseMove = (e) => { + drag = true; + + const cameras = Object.values( + sceneTree.find_no_create([CAMERAS_NAME]).children, + ).map((obj) => obj.object.children[0].children[1]); + + sceneTree.metadata.renderer.getSize(size); + mouseVector.x = 2 * (e.clientX / size.x) - 1; + mouseVector.y = 1 - 2 * ((e.clientY - 50) / size.y); + + // hacky out of bounds fix + if (mouseVector.x > 1 || mouseVector.x < -1) { + // some value that won't hit + mouseVector.x = -100; + } + if (mouseVector.y > 1 || mouseVector.y < -1) { + mouseVector.y = -100; + } + + raycaster.setFromCamera(mouseVector, sceneTree.metadata.camera); + const intersections = raycaster.intersectObjects(cameras, true); + + if (selectedCam !== null) { + selectedCam.material.color = new THREE.Color(1, 1, 1); + selectedCam = null; + } + if (intersections.length > 0) { + selectedCam = intersections[0].object; + selectedCam.material.color = new THREE.Color(0xfab300); + } + }; + + const onMouseUp = () => { + if (drag === true) { + return; + } + if (selectedCam !== null) { + const clickedCam = sceneTree.find_object_no_create([ + CAMERAS_NAME, + selectedCam.name, + ]); + snap_to_camera(sceneTree, sceneTree.metadata.camera, clickedCam.matrix); + } + }; + window.addEventListener('mousedown', onMouseDown, false); + window.addEventListener('mousemove', onMouseMove, false); + window.addEventListener('mouseup', onMouseUp, false); return sceneTree; } diff --git a/nerfstudio/viewer/app/src/modules/Scene/drawing.js b/nerfstudio/viewer/app/src/modules/Scene/drawing.js index c2ae2996a..fb5ef7af5 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/drawing.js +++ b/nerfstudio/viewer/app/src/modules/Scene/drawing.js @@ -100,7 +100,7 @@ export function getCameraWireframe( return lines; } -export function drawCameraImagePlane(width, height, imageString) { +export function drawCameraImagePlane(width, height, imageString, name) { // imageString is the texture as a base64 string const geometry = new THREE.PlaneGeometry(width, height); const material = new THREE.MeshBasicMaterial({ @@ -109,6 +109,7 @@ export function drawCameraImagePlane(width, height, imageString) { const texture = new THREE.TextureLoader().load(imageString); material.map = texture; const plane = new THREE.Mesh(geometry, material); + plane.name = name; return plane; } @@ -116,7 +117,7 @@ function transpose(matrix) { return matrix[0].map((col, i) => matrix.map((row) => row[i])); } -export function drawCamera(camera): THREE.Object3D { +export function drawCamera(camera, name): THREE.Object3D { const group = new THREE.Group(); console.assert( @@ -139,6 +140,7 @@ export function drawCamera(camera): THREE.Object3D { width * 2, height * 2, camera.image, + name, ); group.add(cameraImagePlaneObject); diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx index cf556a720..51368df70 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx @@ -1,6 +1,7 @@ /* eslint-disable react/jsx-props-no-spreading */ import * as React from 'react'; +import * as THREE from 'three'; import { FaTractor } from 'react-icons/fa'; @@ -20,6 +21,7 @@ import { ExpandLess, ExpandMore, TuneRounded, + Videocam, Visibility, VisibilityOff, WidgetsRounded, @@ -30,7 +32,22 @@ import LevaTheme from '../ConfigPanel/leva_theme.json'; import CameraPanel from './CameraPanel'; import { RenderControls } from '../ConfigPanel/ConfigPanel'; import { LogPanel } from '../LogPanel/LogPanel'; -// import { object } from 'prop-types'; + +export const snap_to_camera = (sceneTree, camera, matrix) => { + const mat = new THREE.Matrix4(); + mat.fromArray(matrix.elements); + mat.decompose(camera.position, camera.quaternion, camera.scale); + const unit = new THREE.Vector3(0, 0, -1); + const viewDirection = unit.applyMatrix4(mat); + sceneTree.metadata.camera_controls.setLookAt( + camera.position.x, + camera.position.y, + camera.position.z, + viewDirection.x, + viewDirection.y, + viewDirection.z, + ); +}; interface TabPanelProps { children: React.ReactNode; @@ -66,20 +83,28 @@ function a11yProps(index: number) { interface ListItemProps { name: String; + sceneTree: SceneNode; scene_node: SceneNode; level: Number; groupVisible: Boolean; + canSnap: Boolean; } function MenuItems(props: ListItemProps) { const name = props.name; + const sceneTree = props.sceneTree; const scene_node = props.scene_node; const level = props.level; const groupVisible = props.groupVisible; + const canSnap = props.canSnap; // TODO: sort the keys by string const terminal = Object.keys(scene_node.children).includes(''); + const getCamera = (node) => { + return node.object.children[0]; + }; + const num_children = Object.keys(scene_node.children).length; if (num_children === 0) { return null; @@ -138,6 +163,22 @@ function MenuItems(props: ListItemProps) { + {canSnap && ( + + snap_to_camera( + sceneTree, + sceneTree.metadata.camera, + getCamera(scene_node).matrix, + ) + } + sx={{ mr: 1 }} + > + + + )} + {visible ? : } @@ -165,9 +206,11 @@ function MenuItems(props: ListItemProps) { .map((key) => ( ))} @@ -186,7 +229,13 @@ function ClickableList(props: ClickableListProps) { return ( - + ); }