diff --git a/nerfactory/cameras/camera_paths.py b/nerfactory/cameras/camera_paths.py
index 96b5050ad..c3e18de13 100644
--- a/nerfactory/cameras/camera_paths.py
+++ b/nerfactory/cameras/camera_paths.py
@@ -16,7 +16,7 @@
Code for camera paths.
"""
-from typing import Optional, Tuple
+from typing import Any, Dict, Optional, Tuple
import torch
@@ -24,6 +24,7 @@ import nerfactory.cameras.utils as camera_utils
import nerfactory.utils.poses as pose_utils
from nerfactory.cameras.cameras import Cameras
from nerfactory.cameras.utils import get_interpolated_poses_many
+from nerfactory.viewer.server.utils import three_js_perspective_camera_focal_length
def get_interpolated_camera_path(cameras: Cameras, steps: int) -> Cameras:
@@ -75,9 +76,7 @@ def get_spiral_path(
else:
raise ValueError("Only one of radius or radiuses must be specified.")
- # TODO: don't hardcode this. pass this in
up = camera.camera_to_worlds[0, :3, 2] # scene is z up
- # up = camera.camera_to_world[:3, 1] # this will rotate 90 degrees
focal = torch.min(camera.fx[0], camera.fy[0])
target = torch.tensor([0, 0, -focal], device=camera.device) # camera looking in -z direction
@@ -98,5 +97,44 @@ def get_spiral_path(
for local_c2wh in local_c2whs:
c2wh = torch.matmul(c2wh_global, local_c2wh)
new_c2ws.append(c2wh[:3, :4])
+ new_c2ws = torch.stack(new_c2ws, dim=0)
return Cameras(fx=camera.fx[0], fy=camera.fy[0], cx=camera.cx, cy=camera.cy, camera_to_worlds=new_c2ws)
+
+
+def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
+ """Takes a camera path dictionary and returns a trajectory as a Camera instance.
+
+ Args:
+ camera_path: A dictionary of the camera path information coming from the viewer.
+
+ Returns:
+ A Cameras instance with the camera path.
+ """
+
+ image_height = camera_path["render_height"]
+ image_width = camera_path["render_width"]
+
+ c2ws = []
+ fxs = []
+ fys = []
+ for camera in camera_path["camera_path"]:
+ # pose
+ c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3]
+ c2ws.append(c2w)
+ # field of view
+ fov = camera["fov"]
+ focal_length = three_js_perspective_camera_focal_length(fov, image_height)
+ fxs.append(focal_length)
+ fys.append(focal_length)
+
+ camera_to_worlds = torch.stack(c2ws, dim=0)
+ fx = torch.tensor(fxs)
+ fy = torch.tensor(fys)
+ return Cameras(
+ fx=fx,
+ fy=fy,
+ cx=image_width / 2,
+ cy=image_height / 2,
+ camera_to_worlds=camera_to_worlds,
+ )
diff --git a/nerfactory/cameras/utils.py b/nerfactory/cameras/utils.py
index 804acb93c..61a77039e 100644
--- a/nerfactory/cameras/utils.py
+++ b/nerfactory/cameras/utils.py
@@ -18,6 +18,7 @@ Camera transformation helper code.
"""
import math
+from turtle import width
from typing import List, Literal, Optional, Tuple
import numpy as np
diff --git a/nerfactory/datamanagers/dataloaders.py b/nerfactory/datamanagers/dataloaders.py
index 8292828cd..fd9af9ba5 100644
--- a/nerfactory/datamanagers/dataloaders.py
+++ b/nerfactory/datamanagers/dataloaders.py
@@ -142,11 +142,11 @@ class EvalDataloader(DataLoader):
distortion_params = self.cameras.distortion_params[image_idx]
camera = Cameras(
- fx=self.cameras.fx[image_idx],
- fy=self.cameras.fy[image_idx],
+ fx=self.cameras.fx if isinstance(self.cameras.fx, float) else self.cameras.fx[image_idx : image_idx + 1],
+ fy=self.cameras.fy if isinstance(self.cameras.fy, float) else self.cameras.fy[image_idx : image_idx + 1],
cx=self.cameras.cx,
cy=self.cameras.cy,
- camera_to_worlds=self.cameras.camera_to_worlds[image_idx],
+ camera_to_worlds=self.cameras.camera_to_worlds[image_idx : image_idx + 1],
distortion_params=distortion_params,
camera_type=self.cameras.camera_type,
)
diff --git a/nerfactory/engine/trainer.py b/nerfactory/engine/trainer.py
index b9e6b5928..c2de9ea89 100644
--- a/nerfactory/engine/trainer.py
+++ b/nerfactory/engine/trainer.py
@@ -80,7 +80,7 @@ class Trainer:
self.start_step = 0
# visualizer variable
banner_messages = None
- self.visualizer_state = viewer_utils.VisualizerState(config.viewer)
+ self.visualizer_state = viewer_utils.VisualizerState(config.viewer, config_base_dir=self.config.base_dir)
if config.viewer.enable:
banner_messages = [f"Viewer at: {self.visualizer_state.viewer_url}"]
self.grad_scaler = GradScaler(enabled=self.mixed_precision)
diff --git a/nerfactory/viewer/app/package.json b/nerfactory/viewer/app/package.json
index 1893c518c..ba7bcbaf4 100644
--- a/nerfactory/viewer/app/package.json
+++ b/nerfactory/viewer/app/package.json
@@ -1,7 +1,7 @@
{
"name": "viewer",
"homepage": ".",
- "version": "22-09-11-0",
+ "version": "22-09-13-0",
"private": true,
"dependencies": {
"@emotion/react": "^11.10.4",
diff --git a/nerfactory/viewer/app/src/App.jsx b/nerfactory/viewer/app/src/App.jsx
index 54dd59acb..1d2987107 100644
--- a/nerfactory/viewer/app/src/App.jsx
+++ b/nerfactory/viewer/app/src/App.jsx
@@ -1,37 +1,35 @@
-import React from 'react';
-
import { CssBaseline, ThemeProvider } from '@mui/material';
-import { appTheme } from './themes/theme.ts';
-import { BasicTabs } from './modules/SidePanel/SidePanel';
+import React from 'react';
import {
SceneTreeWebSocketListener,
get_scene_tree,
} from './modules/Scene/Scene';
-import { RenderControls } from './modules/ConfigPanel/ConfigPanel';
-import ViewerWindow from './modules/ViewerWindow/ViewerWindow';
+
import Banner from './modules/Banner';
+import { BasicTabs } from './modules/SidePanel/SidePanel';
+import ViewerWindow from './modules/ViewerWindow/ViewerWindow';
+import { appTheme } from './themes/theme.ts';
export default function App() {
- // The scene tree won't rerender but it will listener to changes
+ // The scene tree won't rerender but it will listen to changes
// from the redux store and draw three.js objects.
+ // In particular, it listens to changes to 'sceneState' coming over the websocket.
const sceneTree = get_scene_tree();
return (
- {/*
- Code that listens for websocket 'write' messages and updates the redux store.
- */}
+ {/* Listens for websocket 'write' messages and updates the redux store. */}
- {/* ----- */}
+ {/* The banner at the top of the page. */}
+ {/* Order matters here. The viewer window must be rendered first. */}
+
-
-
diff --git a/nerfactory/viewer/app/src/SceneNode.js b/nerfactory/viewer/app/src/SceneNode.js
index c0c99244f..1f28d3670 100644
--- a/nerfactory/viewer/app/src/SceneNode.js
+++ b/nerfactory/viewer/app/src/SceneNode.js
@@ -28,7 +28,8 @@ function dispose(object) {
export default class SceneNode {
constructor(object) {
- this.object = object;
+ this.metadata = {}; // relevant metadata for the object
+ this.object = object; // three.js objects
this.children = {};
for (const c of this.object.children) {
this.add_child(c);
@@ -108,7 +109,7 @@ export default class SceneNode {
delete(path) {
if (path.length === 0) {
- console.error("Can't delete an empty path");
+ console.error("Can't delete an empty path.");
} else {
const parent = this.find(path.slice(0, path.length - 1));
const name = path[path.length - 1];
diff --git a/nerfactory/viewer/app/src/index.scss b/nerfactory/viewer/app/src/index.scss
index d751d7772..a34086ea8 100644
--- a/nerfactory/viewer/app/src/index.scss
+++ b/nerfactory/viewer/app/src/index.scss
@@ -216,7 +216,8 @@ $sidebar-width: 100% !default;
// Viewer Window
.canvas-container-main,
-.WebRTCVideo {
+.WebRTCVideo,
+.ViewerWindow-render-crop-container {
position: absolute;
top: 0px;
right: $SidePanel-width;
@@ -225,14 +226,45 @@ $sidebar-width: 100% !default;
border: $custom-border;
}
+.WebRTCVideo {
+ z-index: 1;
+}
+
+.ViewerWindow-render-crop-container {
+ z-index: 2;
+ display: flex;
+ justify-content: center;
+ align-items: center;
+}
+
+.canvas-container-main {
+ z-index: 3;
+}
+
.WebRTCVideo-video {
width: 100%;
height: 100%;
object-fit: fill;
}
+.ViewerWindow-render-crop {
+ display: flex;
+ background-color: black;
+ z-index: 1;
+ opacity: 0.2;
+ vertical-align: middle;
+}
+
$viewer_button_size: 50px;
+.ViewerWindow-camera-dropdown {
+ position: absolute;
+ margin: 10px;
+ top: 0px;
+ right: $SidePanel-width + $viewer_button_size;
+ z-index: 1000;
+}
+
.ViewerWindow-buttons {
position: absolute;
margin: 10px;
@@ -240,7 +272,6 @@ $viewer_button_size: 50px;
right: $SidePanel-width;
z-index: 1000;
width: $viewer_button_size;
- // background-color: $base-color1;
}
.ViewerWindow-iconbutton {
diff --git a/nerfactory/viewer/app/src/modules/Scene/Scene.jsx b/nerfactory/viewer/app/src/modules/Scene/Scene.jsx
index 13c2530f7..f769e2634 100644
--- a/nerfactory/viewer/app/src/modules/Scene/Scene.jsx
+++ b/nerfactory/viewer/app/src/modules/Scene/Scene.jsx
@@ -2,9 +2,12 @@
import * as THREE from 'three';
import { useContext, useEffect } from 'react';
+import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls';
+import { TransformControls } from 'three/examples/jsm/controls/TransformControls';
import { useDispatch } from 'react-redux';
import { drawCamera, drawSceneBounds } from './drawing';
+import { CameraHelper } from '../SidePanel/CameraPanel/CameraHelper';
import SceneNode from '../../SceneNode';
import { WebSocketContext } from '../WebSocket/WebSocket';
import { subscribe_to_changes } from '../../subscriber';
@@ -15,37 +18,92 @@ const SCENE_BOUNDS_NAME = 'Scene Bounds';
const CAMERAS_NAME = 'Training Cameras';
export function get_scene_tree() {
- let scene = null;
- let sceneTree = null;
- scene = new THREE.Scene();
- sceneTree = new SceneNode(scene);
+ const scene = new THREE.Scene();
+ const sceneTree = new SceneNode(scene);
- // add objects to the the scene tree
- const setObject = (path, object) => {
- sceneTree.find(path.concat(['