Snap to cameras (#550)

Clicking on training cameras now snaps to them

Co-authored-by: mcallisterdavid <davidrmc2@gmail.com>

nerfstudio/viewer/app/src/modules/Scene/Scene.jsx

@@ -13,6 +13,7 @@ import { CameraHelper } from '../SidePanel/CameraPanel/CameraHelper';
 import SceneNode from '../../SceneNode';
 import { WebSocketContext } from '../WebSocket/WebSocket';
 import { subscribe_to_changes } from '../../subscriber';
+import { snap_to_camera } from '../SidePanel/SidePanel';
 
 const msgpack = require('msgpack-lite');
 
@@ -203,7 +204,7 @@ export function get_scene_tree() {
         if (!prev.has(key)) {
           // keys_valid.push(key);
           const json = current[key];
-          const camera = drawCamera(json);
+          const camera = drawCamera(json, key);
           sceneTree.set_object_from_path([CAMERAS_NAME, key], camera);
         }
       }
@@ -220,6 +221,65 @@ export function get_scene_tree() {
   };
   subscribe_to_changes(selector_fn_cameras, fn_value_cameras);
 
+  // Check for clicks on training cameras
+  const mouseVector = new THREE.Vector2();
+  const raycaster = new THREE.Raycaster();
+  const size = new THREE.Vector2();
+  let selectedCam = null;
+
+  let drag = false;
+  const onMouseDown = () => {
+    drag = false;
+  };
+
+  const onMouseMove = (e) => {
+    drag = true;
+
+    const cameras = Object.values(
+      sceneTree.find_no_create([CAMERAS_NAME]).children,
+    ).map((obj) => obj.object.children[0].children[1]);
+
+    sceneTree.metadata.renderer.getSize(size);
+    mouseVector.x = 2 * (e.clientX / size.x) - 1;
+    mouseVector.y = 1 - 2 * ((e.clientY - 50) / size.y);
+
+    // hacky out of bounds fix
+    if (mouseVector.x > 1 || mouseVector.x < -1) {
+      // some value that won't hit
+      mouseVector.x = -100;
+    }
+    if (mouseVector.y > 1 || mouseVector.y < -1) {
+      mouseVector.y = -100;
+    }
+
+    raycaster.setFromCamera(mouseVector, sceneTree.metadata.camera);
+    const intersections = raycaster.intersectObjects(cameras, true);
+
+    if (selectedCam !== null) {
+      selectedCam.material.color = new THREE.Color(1, 1, 1);
+      selectedCam = null;
+    }
+    if (intersections.length > 0) {
+      selectedCam = intersections[0].object;
+      selectedCam.material.color = new THREE.Color(0xfab300);
+    }
+  };
+
+  const onMouseUp = () => {
+    if (drag === true) {
+      return;
+    }
+    if (selectedCam !== null) {
+      const clickedCam = sceneTree.find_object_no_create([
+        CAMERAS_NAME,
+        selectedCam.name,
+      ]);
+      snap_to_camera(sceneTree, sceneTree.metadata.camera, clickedCam.matrix);
+    }
+  };
+  window.addEventListener('mousedown', onMouseDown, false);
+  window.addEventListener('mousemove', onMouseMove, false);
+  window.addEventListener('mouseup', onMouseUp, false);
   return sceneTree;
 }
 

nerfstudio/viewer/app/src/modules/Scene/drawing.js

@@ -100,7 +100,7 @@ export function getCameraWireframe(
   return lines;
 }
 
-export function drawCameraImagePlane(width, height, imageString) {
+export function drawCameraImagePlane(width, height, imageString, name) {
   // imageString is the texture as a base64 string
   const geometry = new THREE.PlaneGeometry(width, height);
   const material = new THREE.MeshBasicMaterial({
@@ -109,6 +109,7 @@ export function drawCameraImagePlane(width, height, imageString) {
   const texture = new THREE.TextureLoader().load(imageString);
   material.map = texture;
   const plane = new THREE.Mesh(geometry, material);
+  plane.name = name;
   return plane;
 }
 
@@ -116,7 +117,7 @@ function transpose(matrix) {
   return matrix[0].map((col, i) => matrix.map((row) => row[i]));
 }
 
-export function drawCamera(camera): THREE.Object3D {
+export function drawCamera(camera, name): THREE.Object3D {
   const group = new THREE.Group();
 
   console.assert(
@@ -139,6 +140,7 @@ export function drawCamera(camera): THREE.Object3D {
     width * 2,
     height * 2,
     camera.image,
+    name,
   );
   group.add(cameraImagePlaneObject);
 

nerfstudio/viewer/app/src/modules/SidePanel/SidePanel.jsx

@@ -1,6 +1,7 @@
 /* eslint-disable react/jsx-props-no-spreading */
 
 import * as React from 'react';
+import * as THREE from 'three';
 
 import { FaTractor } from 'react-icons/fa';
 
@@ -20,6 +21,7 @@ import {
   ExpandLess,
   ExpandMore,
   TuneRounded,
+  Videocam,
   Visibility,
   VisibilityOff,
   WidgetsRounded,
@@ -30,7 +32,22 @@ import LevaTheme from '../ConfigPanel/leva_theme.json';
 import CameraPanel from './CameraPanel';
 import { RenderControls } from '../ConfigPanel/ConfigPanel';
 import { LogPanel } from '../LogPanel/LogPanel';
-// import { object } from 'prop-types';
+
+export const snap_to_camera = (sceneTree, camera, matrix) => {
+  const mat = new THREE.Matrix4();
+  mat.fromArray(matrix.elements);
+  mat.decompose(camera.position, camera.quaternion, camera.scale);
+  const unit = new THREE.Vector3(0, 0, -1);
+  const viewDirection = unit.applyMatrix4(mat);
+  sceneTree.metadata.camera_controls.setLookAt(
+    camera.position.x,
+    camera.position.y,
+    camera.position.z,
+    viewDirection.x,
+    viewDirection.y,
+    viewDirection.z,
+  );
+};
 
 interface TabPanelProps {
   children: React.ReactNode;
@@ -66,20 +83,28 @@ function a11yProps(index: number) {
 
 interface ListItemProps {
   name: String;
+  sceneTree: SceneNode;
   scene_node: SceneNode;
   level: Number;
   groupVisible: Boolean;
+  canSnap: Boolean;
 }
 
 function MenuItems(props: ListItemProps) {
   const name = props.name;
+  const sceneTree = props.sceneTree;
   const scene_node = props.scene_node;
   const level = props.level;
   const groupVisible = props.groupVisible;
+  const canSnap = props.canSnap;
 
   // TODO: sort the keys by string
   const terminal = Object.keys(scene_node.children).includes('<object>');
 
+  const getCamera = (node) => {
+    return node.object.children[0];
+  };
+
   const num_children = Object.keys(scene_node.children).length;
   if (num_children === 0) {
     return null;
@@ -138,6 +163,22 @@ function MenuItems(props: ListItemProps) {
         </ListItemIcon>
         <ListItemText primary={name} />
 
+        {canSnap && (
+          <IconButton
+            aria-label="visibility"
+            onClick={() =>
+              snap_to_camera(
+                sceneTree,
+                sceneTree.metadata.camera,
+                getCamera(scene_node).matrix,
+              )
+            }
+            sx={{ mr: 1 }}
+          >
+            <Videocam />
+          </IconButton>
+        )}
+
         <IconButton aria-label="visibility" onClick={toggleVisible}>
           {visible ? <Visibility /> : <VisibilityOff />}
         </IconButton>
@@ -165,9 +206,11 @@ function MenuItems(props: ListItemProps) {
               .map((key) => (
                 <MenuItems
                   name={key}
+                  sceneTree={sceneTree}
                   scene_node={scene_node.children[key]}
                   level={level + 1}
                   groupVisible={visible}
+                  canSnap={name === 'Training Cameras'}
                 />
               ))}
           </List>
@@ -186,7 +229,13 @@ function ClickableList(props: ClickableListProps) {
 
   return (
     <List sx={{ color: LevaTheme.colors.accent2 }}>
-      <MenuItems name="Scene" scene_node={sceneTree} level={0} groupVisible />
+      <MenuItems
+        name="Scene"
+        sceneTree={sceneTree}
+        scene_node={sceneTree}
+        level={0}
+        groupVisible
+      />
     </List>
   );
 }