diff --git a/nerfactory/cameras/camera_paths.py b/nerfactory/cameras/camera_paths.py index 96b5050ad..c3e18de13 100644 --- a/nerfactory/cameras/camera_paths.py +++ b/nerfactory/cameras/camera_paths.py @@ -16,7 +16,7 @@ Code for camera paths. """ -from typing import Optional, Tuple +from typing import Any, Dict, Optional, Tuple import torch @@ -24,6 +24,7 @@ import nerfactory.cameras.utils as camera_utils import nerfactory.utils.poses as pose_utils from nerfactory.cameras.cameras import Cameras from nerfactory.cameras.utils import get_interpolated_poses_many +from nerfactory.viewer.server.utils import three_js_perspective_camera_focal_length def get_interpolated_camera_path(cameras: Cameras, steps: int) -> Cameras: @@ -75,9 +76,7 @@ def get_spiral_path( else: raise ValueError("Only one of radius or radiuses must be specified.") - # TODO: don't hardcode this. pass this in up = camera.camera_to_worlds[0, :3, 2] # scene is z up - # up = camera.camera_to_world[:3, 1] # this will rotate 90 degrees focal = torch.min(camera.fx[0], camera.fy[0]) target = torch.tensor([0, 0, -focal], device=camera.device) # camera looking in -z direction @@ -98,5 +97,44 @@ def get_spiral_path( for local_c2wh in local_c2whs: c2wh = torch.matmul(c2wh_global, local_c2wh) new_c2ws.append(c2wh[:3, :4]) + new_c2ws = torch.stack(new_c2ws, dim=0) return Cameras(fx=camera.fx[0], fy=camera.fy[0], cx=camera.cx, cy=camera.cy, camera_to_worlds=new_c2ws) + + +def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras: + """Takes a camera path dictionary and returns a trajectory as a Camera instance. + + Args: + camera_path: A dictionary of the camera path information coming from the viewer. + + Returns: + A Cameras instance with the camera path. + """ + + image_height = camera_path["render_height"] + image_width = camera_path["render_width"] + + c2ws = [] + fxs = [] + fys = [] + for camera in camera_path["camera_path"]: + # pose + c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3] + c2ws.append(c2w) + # field of view + fov = camera["fov"] + focal_length = three_js_perspective_camera_focal_length(fov, image_height) + fxs.append(focal_length) + fys.append(focal_length) + + camera_to_worlds = torch.stack(c2ws, dim=0) + fx = torch.tensor(fxs) + fy = torch.tensor(fys) + return Cameras( + fx=fx, + fy=fy, + cx=image_width / 2, + cy=image_height / 2, + camera_to_worlds=camera_to_worlds, + ) diff --git a/nerfactory/cameras/utils.py b/nerfactory/cameras/utils.py index 804acb93c..61a77039e 100644 --- a/nerfactory/cameras/utils.py +++ b/nerfactory/cameras/utils.py @@ -18,6 +18,7 @@ Camera transformation helper code. """ import math +from turtle import width from typing import List, Literal, Optional, Tuple import numpy as np diff --git a/nerfactory/datamanagers/dataloaders.py b/nerfactory/datamanagers/dataloaders.py index 8292828cd..fd9af9ba5 100644 --- a/nerfactory/datamanagers/dataloaders.py +++ b/nerfactory/datamanagers/dataloaders.py @@ -142,11 +142,11 @@ class EvalDataloader(DataLoader): distortion_params = self.cameras.distortion_params[image_idx] camera = Cameras( - fx=self.cameras.fx[image_idx], - fy=self.cameras.fy[image_idx], + fx=self.cameras.fx if isinstance(self.cameras.fx, float) else self.cameras.fx[image_idx : image_idx + 1], + fy=self.cameras.fy if isinstance(self.cameras.fy, float) else self.cameras.fy[image_idx : image_idx + 1], cx=self.cameras.cx, cy=self.cameras.cy, - camera_to_worlds=self.cameras.camera_to_worlds[image_idx], + camera_to_worlds=self.cameras.camera_to_worlds[image_idx : image_idx + 1], distortion_params=distortion_params, camera_type=self.cameras.camera_type, ) diff --git a/nerfactory/engine/trainer.py b/nerfactory/engine/trainer.py index b9e6b5928..c2de9ea89 100644 --- a/nerfactory/engine/trainer.py +++ b/nerfactory/engine/trainer.py @@ -80,7 +80,7 @@ class Trainer: self.start_step = 0 # visualizer variable banner_messages = None - self.visualizer_state = viewer_utils.VisualizerState(config.viewer) + self.visualizer_state = viewer_utils.VisualizerState(config.viewer, config_base_dir=self.config.base_dir) if config.viewer.enable: banner_messages = [f"Viewer at: {self.visualizer_state.viewer_url}"] self.grad_scaler = GradScaler(enabled=self.mixed_precision) diff --git a/nerfactory/viewer/app/package.json b/nerfactory/viewer/app/package.json index 1893c518c..ba7bcbaf4 100644 --- a/nerfactory/viewer/app/package.json +++ b/nerfactory/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "22-09-11-0", + "version": "22-09-13-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfactory/viewer/app/src/App.jsx b/nerfactory/viewer/app/src/App.jsx index 54dd59acb..1d2987107 100644 --- a/nerfactory/viewer/app/src/App.jsx +++ b/nerfactory/viewer/app/src/App.jsx @@ -1,37 +1,35 @@ -import React from 'react'; - import { CssBaseline, ThemeProvider } from '@mui/material'; -import { appTheme } from './themes/theme.ts'; -import { BasicTabs } from './modules/SidePanel/SidePanel'; +import React from 'react'; import { SceneTreeWebSocketListener, get_scene_tree, } from './modules/Scene/Scene'; -import { RenderControls } from './modules/ConfigPanel/ConfigPanel'; -import ViewerWindow from './modules/ViewerWindow/ViewerWindow'; + import Banner from './modules/Banner'; +import { BasicTabs } from './modules/SidePanel/SidePanel'; +import ViewerWindow from './modules/ViewerWindow/ViewerWindow'; +import { appTheme } from './themes/theme.ts'; export default function App() { - // The scene tree won't rerender but it will listener to changes + // The scene tree won't rerender but it will listen to changes // from the redux store and draw three.js objects. + // In particular, it listens to changes to 'sceneState' coming over the websocket. const sceneTree = get_scene_tree(); return (
- {/* - Code that listens for websocket 'write' messages and updates the redux store. - */} + {/* Listens for websocket 'write' messages and updates the redux store. */} - {/* ----- */} + {/* The banner at the top of the page. */}
+ {/* Order matters here. The viewer window must be rendered first. */} +
- -
diff --git a/nerfactory/viewer/app/src/SceneNode.js b/nerfactory/viewer/app/src/SceneNode.js index c0c99244f..1f28d3670 100644 --- a/nerfactory/viewer/app/src/SceneNode.js +++ b/nerfactory/viewer/app/src/SceneNode.js @@ -28,7 +28,8 @@ function dispose(object) { export default class SceneNode { constructor(object) { - this.object = object; + this.metadata = {}; // relevant metadata for the object + this.object = object; // three.js objects this.children = {}; for (const c of this.object.children) { this.add_child(c); @@ -108,7 +109,7 @@ export default class SceneNode { delete(path) { if (path.length === 0) { - console.error("Can't delete an empty path"); + console.error("Can't delete an empty path."); } else { const parent = this.find(path.slice(0, path.length - 1)); const name = path[path.length - 1]; diff --git a/nerfactory/viewer/app/src/index.scss b/nerfactory/viewer/app/src/index.scss index d751d7772..a34086ea8 100644 --- a/nerfactory/viewer/app/src/index.scss +++ b/nerfactory/viewer/app/src/index.scss @@ -216,7 +216,8 @@ $sidebar-width: 100% !default; // Viewer Window .canvas-container-main, -.WebRTCVideo { +.WebRTCVideo, +.ViewerWindow-render-crop-container { position: absolute; top: 0px; right: $SidePanel-width; @@ -225,14 +226,45 @@ $sidebar-width: 100% !default; border: $custom-border; } +.WebRTCVideo { + z-index: 1; +} + +.ViewerWindow-render-crop-container { + z-index: 2; + display: flex; + justify-content: center; + align-items: center; +} + +.canvas-container-main { + z-index: 3; +} + .WebRTCVideo-video { width: 100%; height: 100%; object-fit: fill; } +.ViewerWindow-render-crop { + display: flex; + background-color: black; + z-index: 1; + opacity: 0.2; + vertical-align: middle; +} + $viewer_button_size: 50px; +.ViewerWindow-camera-dropdown { + position: absolute; + margin: 10px; + top: 0px; + right: $SidePanel-width + $viewer_button_size; + z-index: 1000; +} + .ViewerWindow-buttons { position: absolute; margin: 10px; @@ -240,7 +272,6 @@ $viewer_button_size: 50px; right: $SidePanel-width; z-index: 1000; width: $viewer_button_size; - // background-color: $base-color1; } .ViewerWindow-iconbutton { diff --git a/nerfactory/viewer/app/src/modules/Scene/Scene.jsx b/nerfactory/viewer/app/src/modules/Scene/Scene.jsx index 13c2530f7..f769e2634 100644 --- a/nerfactory/viewer/app/src/modules/Scene/Scene.jsx +++ b/nerfactory/viewer/app/src/modules/Scene/Scene.jsx @@ -2,9 +2,12 @@ import * as THREE from 'three'; import { useContext, useEffect } from 'react'; +import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; +import { TransformControls } from 'three/examples/jsm/controls/TransformControls'; import { useDispatch } from 'react-redux'; import { drawCamera, drawSceneBounds } from './drawing'; +import { CameraHelper } from '../SidePanel/CameraPanel/CameraHelper'; import SceneNode from '../../SceneNode'; import { WebSocketContext } from '../WebSocket/WebSocket'; import { subscribe_to_changes } from '../../subscriber'; @@ -15,37 +18,92 @@ const SCENE_BOUNDS_NAME = 'Scene Bounds'; const CAMERAS_NAME = 'Training Cameras'; export function get_scene_tree() { - let scene = null; - let sceneTree = null; - scene = new THREE.Scene(); - sceneTree = new SceneNode(scene); + const scene = new THREE.Scene(); + const sceneTree = new SceneNode(scene); - // add objects to the the scene tree - const setObject = (path, object) => { - sceneTree.find(path.concat([''])).set_object(object); - }; - const deleteObject = (path) => { - sceneTree.delete(path); - }; + const dispatch = useDispatch(); // Main camera const main_camera = new THREE.PerspectiveCamera(75, 1, 0.1, 1000); - setObject(['Main Camera'], main_camera); + main_camera.position.x = 5; + main_camera.position.y = -5; + main_camera.position.z = 5; + main_camera.up = new THREE.Vector3(0, 0, 1); + sceneTree.set_object_from_path(['Cameras', 'Main Camera'], main_camera); + + sceneTree.set_object_from_path( + ['Cameras', 'Main Camera', 'Helper'], + new CameraHelper(main_camera), + ); + sceneTree.metadata.camera = main_camera; + + // Render camera + const render_camera = main_camera.clone(); + sceneTree.set_object_from_path(['Cameras', 'Render Camera'], render_camera); + sceneTree.set_object_from_path( + ['Cameras', 'Render Camera', 'Helper'], + new CameraHelper(render_camera), + ); + + // Renderer + const renderer = new THREE.WebGLRenderer({ + antialias: true, + alpha: true, + }); + renderer.setPixelRatio(window.devicePixelRatio); + sceneTree.metadata.renderer = renderer; + + // Camera Controls + const camera_controls = new OrbitControls(main_camera, renderer.domElement); + camera_controls.rotateSpeed = 2.0; + camera_controls.zoomSpeed = 0.3; + camera_controls.panSpeed = 0.2; + camera_controls.target.set(0, 0, 0); // focus point of the controls + camera_controls.autoRotate = false; + camera_controls.enableDamping = true; + camera_controls.dampingFactor = 1.0; + sceneTree.metadata.camera_controls = camera_controls; + + // if you drag the screen when the render camera is shown, + // then snap back to the main camera + // eslint-disable-next-line no-unused-vars + camera_controls.addEventListener('change', (event) => { + if (sceneTree.metadata.camera === render_camera) { + dispatch({ + type: 'write', + path: 'renderingState/camera_choice', + data: 'Main Camera', + }); + } + }); + + // Listen for changes to the camera + + // Transform Controls + const transformsControls = new TransformControls( + main_camera, + renderer.domElement, + ); + sceneTree.metadata.transformsControls = transformsControls; + + transformsControls.addEventListener('dragging-changed', (event) => { + camera_controls.enabled = !event.value; + }); // Axes const axes = new THREE.AxesHelper(5); - setObject(['Axes'], axes); + sceneTree.set_object_from_path(['Axes'], axes); // Grid const grid = new THREE.GridHelper(20, 20); grid.rotateX(Math.PI / 2); // rotated to xy plane - setObject(['Grid'], grid); + sceneTree.set_object_from_path(['Grid'], grid); // Lights const color = 0xffffff; const intensity = 1; const light = new THREE.AmbientLight(color, intensity); - setObject(['Light'], light); + sceneTree.set_object_from_path(['Light'], light); // draw scene bounds const selector_fn_scene_bounds = (state) => { @@ -54,9 +112,9 @@ export function get_scene_tree() { const fn_value_scene_bounds = (previous, current) => { if (current !== null) { const line = drawSceneBounds(current); - setObject([SCENE_BOUNDS_NAME], line); + sceneTree.set_object_from_path([SCENE_BOUNDS_NAME], line); } else { - deleteObject([SCENE_BOUNDS_NAME]); + sceneTree.delete([SCENE_BOUNDS_NAME]); } }; subscribe_to_changes(selector_fn_scene_bounds, fn_value_scene_bounds); @@ -82,18 +140,18 @@ export function get_scene_tree() { // keys_valid.push(key); const json = current[key]; const camera = drawCamera(json); - setObject([CAMERAS_NAME, key], camera); + sceneTree.set_object_from_path([CAMERAS_NAME, key], camera); } } for (const key of prev) { // invalid so delete if (!curr.has(key) || current[key] === null) { // keys_invalid.push(key); - deleteObject([CAMERAS_NAME, key]); + sceneTree.delete([CAMERAS_NAME, key]); } } } else { - deleteObject([CAMERAS_NAME]); + sceneTree.delete([CAMERAS_NAME]); } }; subscribe_to_changes(selector_fn_cameras, fn_value_cameras); diff --git a/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/CameraHelper.js b/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/CameraHelper.js index 7a3d3cb55..7fb843809 100644 --- a/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/CameraHelper.js +++ b/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/CameraHelper.js @@ -30,7 +30,7 @@ class CameraHelper extends THREE.Mesh { const geometry = new THREE.BufferGeometry(); const material = new MeshLineMaterial({ color: 0x000000, - lineWidth: 0.04, + lineWidth: 0.01, }); const vertices = []; @@ -106,25 +106,22 @@ class CameraHelper extends THREE.Mesh { // we need just camera projection matrix inverse // world matrix must be identity + console.log(_camera.projectionMatrixInverse); + _camera.projectionMatrixInverse.copy(this.camera.projectionMatrixInverse); - // center - - setPoint('c', pointMap, geometry, _camera, 0, 0, -1); - setPoint('t', pointMap, geometry, _camera, 0, 0, 1); - // near - setPoint('n1', pointMap, geometry, _camera, -w, -h, z); - setPoint('n2', pointMap, geometry, _camera, w, -h, z); - setPoint('n3', pointMap, geometry, _camera, -w, h, z); - setPoint('n4', pointMap, geometry, _camera, w, h, z); + setPoint('n1', pointMap, geometry, _camera, -w, -h, -z); + setPoint('n2', pointMap, geometry, _camera, w, -h, -z); + setPoint('n3', pointMap, geometry, _camera, -w, h, -z); + setPoint('n4', pointMap, geometry, _camera, w, h, -z); // up - setPoint('u1', pointMap, geometry, _camera, w * 0.7, h * 1.1, z); - setPoint('u2', pointMap, geometry, _camera, -w * 0.7, h * 1.1, z); - setPoint('u3', pointMap, geometry, _camera, 0, h * 2, z); + setPoint('u1', pointMap, geometry, _camera, w * 0.7, h * 1.1, -z); + setPoint('u2', pointMap, geometry, _camera, -w * 0.7, h * 1.1, -z); + setPoint('u3', pointMap, geometry, _camera, 0, h * 2, -z); geometry.getAttribute('position').needsUpdate = true; diff --git a/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index 824d93ab0..13e3f14eb 100644 --- a/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -1,23 +1,31 @@ import * as React from 'react'; import * as THREE from 'three'; -import { MeshLine, MeshLineMaterial } from 'meshline'; import { Button, Slider } from '@mui/material'; +import { MeshLine, MeshLineMaterial } from 'meshline'; +import { useContext, useEffect } from 'react'; +import { useDispatch, useSelector } from 'react-redux'; import ArrowBackIosNewIcon from '@mui/icons-material/ArrowBackIosNew'; import ArrowForwardIosIcon from '@mui/icons-material/ArrowForwardIos'; +import ContentPasteGoIcon from '@mui/icons-material/ContentPasteGo'; import DeleteIcon from '@mui/icons-material/Delete'; import FirstPageIcon from '@mui/icons-material/FirstPage'; +import IconButton from '@mui/material/IconButton'; import LastPageIcon from '@mui/icons-material/LastPage'; import PauseIcon from '@mui/icons-material/Pause'; import PlayArrowIcon from '@mui/icons-material/PlayArrow'; import TextField from '@mui/material/TextField'; +import Tooltip from '@mui/material/Tooltip'; import VisibilityIcon from '@mui/icons-material/Visibility'; -import { useEffect } from 'react'; import { CameraHelper } from './CameraHelper'; -import { get_curve_object_from_cameras } from './curve'; +import { get_curve_object_from_cameras, get_transform_matrix } from './curve'; +import { WebSocketContext } from '../../WebSocket/WebSocket'; + +const msgpack = require('msgpack-lite'); function set_camera_position(camera, matrix) { + // console.log("setting camera position"); const mat = new THREE.Matrix4(); mat.fromArray(matrix.elements); mat.decompose(camera.position, camera.quaternion, camera.scale); @@ -70,19 +78,55 @@ function CameraList(props) { } export default function CameraPanel(props) { - // unpack props + // unpack relevant information const sceneTree = props.sceneTree; - - // scene tree objects - const camera_main = sceneTree.find_object(['Main Camera']); + const camera_main = sceneTree.find_object(['Cameras', 'Main Camera']); + const camera_render = sceneTree.find_object(['Cameras', 'Render Camera']); const transform_controls = sceneTree.find_object(['Transform Controls']); + // redux store state + const config_base_dir = useSelector( + (state) => state.renderingState.config_base_dir, + ); + const websocket = useContext(WebSocketContext).socket; + // react state const [cameras, setCameras] = React.useState([]); const [slider_value, set_slider_value] = React.useState(0); const [is_playing, setIsPlaying] = React.useState(false); const [seconds, setSeconds] = React.useState(4); const [fps, setFps] = React.useState(24); + + // const [render_height, setRenderHeight] = React.useState(1080); + // const [render_width, setRenderWidth] = React.useState(1920); + + const dispatch = useDispatch(); + const render_height = useSelector( + (state) => state.renderingState.render_height, + ); + const render_width = useSelector( + (state) => state.renderingState.render_width, + ); + + const field_of_view = useSelector( + (state) => state.renderingState.field_of_view, + ); + + const setRenderHeight = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/render_height', + data: value, + }); + }; + const setRenderWidth = (value) => { + dispatch({ + type: 'write', + path: 'renderingState/render_width', + data: value, + }); + }; + const total_num_steps = seconds * fps; const step_size = (cameras.length - 1) / total_num_steps; const slider_min = 0; @@ -90,9 +134,9 @@ export default function CameraPanel(props) { const add_camera = () => { const camera_main_copy = camera_main.clone(); - camera_main_copy.far = camera_main_copy.near + 0.1; const new_camera_list = cameras.concat(camera_main_copy); setCameras(new_camera_list); + set_camera_position(camera_render, camera_main_copy.matrix); }; // force a rerender if the cameras are dragged around @@ -121,9 +165,7 @@ export default function CameraPanel(props) { sceneTree.delete(['Camera Path', 'Cameras']); // delete old cameras, which is important for (let i = 0; i < cameras.length; i += 1) { const camera = cameras[i]; - const helper = new CameraHelper(camera); - console.log('HERERE'); - console.log(helper); + const camera_helper = new CameraHelper(camera); // camera sceneTree.set_object_from_path( ['Camera Path', 'Cameras', i.toString(), 'Camera'], @@ -132,7 +174,7 @@ export default function CameraPanel(props) { // camera helper sceneTree.set_object_from_path( ['Camera Path', 'Cameras', i.toString(), 'Camera Helper'], - helper, + camera_helper, ); } }, [cameras]); @@ -162,36 +204,15 @@ export default function CameraPanel(props) { const position = curve_object.curve_positions.getPoint( slider_value / (cameras.length - 1.0), ); - const up = curve_object.curve_ups.getPoint(slider_value / cameras.length); const lookat = curve_object.curve_lookats.getPoint( - slider_value / cameras.length, + slider_value / (cameras.length - 1.0), + ); + const up = curve_object.curve_ups.getPoint( + slider_value / (cameras.length - 1.0), ); - // create a copy of the vector up - const up_copy = up.clone(); - const cross = up_copy.cross(lookat); - - up.normalize(); - lookat.normalize(); - cross.normalize(); - - // create the camera transform matrix - const mat = new THREE.Matrix4(); - mat.set( - cross.x, - up.x, - lookat.x, - position.x, - cross.y, - up.y, - lookat.y, - position.y, - cross.z, - up.z, - lookat.z, - position.z, - ); - set_camera_position(camera_main, mat); + const mat = get_transform_matrix(position, lookat, up); + set_camera_position(camera_render, mat); } }, [slider_value]); @@ -214,6 +235,105 @@ export default function CameraPanel(props) { } }, [slider_value]); + const get_camera_path = () => { + // NOTE: currently assuming these are ints + const num_points = fps * seconds; + const positions = curve_object.curve_positions.getPoints(num_points); + const lookats = curve_object.curve_lookats.getPoints(num_points); + const ups = curve_object.curve_ups.getPoints(num_points); + + const camera_path = []; + + for (let i = 0; i < num_points; i += 1) { + const position = positions[i]; + const lookat = lookats[i]; + const up = ups[i]; + + const mat = get_transform_matrix(position, lookat, up); + + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov: camera_render.fov, + aspect: camera_render.aspect, + }); + } + + // const myData + const camera_path_object = { + keyframes: [], + render_height, + render_width, + camera_path, + fps, + seconds, + }; + return camera_path_object; + }; + + const export_camera_path = () => { + // export the camera path + // inspired by: + // https://stackoverflow.com/questions/55613438/reactwrite-to-json-file-or-export-download-no-server + + const camera_path_object = get_camera_path(); + console.log(camera_render.toJSON()); + + // create file in browser + const json = JSON.stringify(camera_path_object, null, 2); + const blob = new Blob([json], { type: 'application/json' }); + const href = URL.createObjectURL(blob); + + // create "a" HTLM element with href to file + const link = document.createElement('a'); + link.href = href; + + const filename = 'camera_path.json'; + link.download = filename; + document.body.appendChild(link); + link.click(); + // clean up "a" element & remove ObjectURL + document.body.removeChild(link); + URL.revokeObjectURL(href); + }; + + const copy_cmd_to_clipboard = () => { + console.log('copy_cmd_to_clipboard'); + + const camera_path_object = get_camera_path(); + + // Copy the text inside the text field + const config_filename = `${config_base_dir}/config.yml`; + const camera_path_filename = `${config_base_dir}/camera_path.json`; + const cmd = + `python scripts/run_eval.py render-trajectory --load-config ${config_filename} --traj filename --camera-path-filename ${camera_path_filename} --output-path output.mp4` + + ` --downscale-factor 4`; + navigator.clipboard.writeText(cmd); + + const camera_path_payload = { + camera_path_filename, + camera_path: camera_path_object, + }; + + // send a command of the websocket to save the trajectory somewhere! + if (websocket.readyState === WebSocket.OPEN) { + const data = { + type: 'write', + path: 'camera_path_payload', + data: camera_path_payload, + }; + const message = msgpack.encode(data); + websocket.send(message); + } + }; + + const setFOV = (fov) => { + dispatch({ + type: 'write', + path: 'renderingState/field_of_view', + data: fov, + }); + }; + return (
@@ -222,10 +342,21 @@ export default function CameraPanel(props) {
-
+
+ + + + + +
+
+ { + if (e.target.validity.valid) { + setRenderHeight(parseInt(e.target.value, 10)); + } + }} + value={render_height} + error={render_height <= 0} + helperText={render_height <= 0 ? 'Required' : ''} + variant="standard" + /> + { + if (e.target.validity.valid) { + setRenderWidth(parseInt(e.target.value, 10)); + } + }} + value={render_width} + error={render_width <= 0} + helperText={render_width <= 0 ? 'Required' : ''} + variant="standard" + /> + { + if (e.target.validity.valid) { + setFOV(parseInt(e.target.value, 10)); + } + }} + value={field_of_view} + error={field_of_view <= 0} + helperText={field_of_view <= 0 ? 'Required' : ''} + variant="standard" + /> +
{ if (e.target.validity.valid) { - setSeconds(e.target.value); + setSeconds(parseInt(e.target.value, 10)); } }} value={seconds} @@ -311,7 +495,7 @@ export default function CameraPanel(props) { size="small" onChange={(e) => { if (e.target.validity.valid) { - setFps(e.target.value); + setFps(parseInt(e.target.value, 10)); } }} value={fps} @@ -324,7 +508,7 @@ export default function CameraPanel(props) { diff --git a/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/curve.js b/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/curve.js index b59c14826..ccc53c707 100644 --- a/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/curve.js +++ b/nerfactory/viewer/app/src/modules/SidePanel/CameraPanel/curve.js @@ -2,12 +2,13 @@ import * as THREE from 'three'; + function get_curve(list_of_3d_vectors) { // TODO: add some hyperparameters to this function const curve = new THREE.CatmullRomCurve3( list_of_3d_vectors, false, - 'centripetal', + 'catmullrom', ); return curve; } @@ -25,7 +26,8 @@ export function get_curve_object_from_cameras(cameras) { const camera = cameras[i]; const up = new THREE.Vector3(0, 1, 0); // y is up in local space - const lookat = new THREE.Vector3(0, 0, -1); // -z is forward in local space + const lookat = new THREE.Vector3(0, 0, 1); // z is forward in local space + up.applyQuaternion(camera.quaternion); lookat.applyQuaternion(camera.quaternion); @@ -50,3 +52,36 @@ export function get_curve_object_from_cameras(cameras) { return curve_object; } + +export function get_transform_matrix(position, lookat, up) { + // normalize the vectors + lookat.normalize(); + up.normalize(); + + // create a copy of the vector up + const up_copy = up.clone(); + const cross = up_copy.cross(lookat); + cross.normalize(); + + // create the camera transform matrix + const mat = new THREE.Matrix4(); + mat.set( + cross.x, + up.x, + lookat.x, + position.x, + cross.y, + up.y, + lookat.y, + position.y, + cross.z, + up.z, + lookat.z, + position.z, + 0, + 0, + 0, + 1, + ); + return mat; +} diff --git a/nerfactory/viewer/app/src/modules/SidePanel/SidePanel.jsx b/nerfactory/viewer/app/src/modules/SidePanel/SidePanel.jsx index 497707af5..041583e9d 100644 --- a/nerfactory/viewer/app/src/modules/SidePanel/SidePanel.jsx +++ b/nerfactory/viewer/app/src/modules/SidePanel/SidePanel.jsx @@ -5,21 +5,21 @@ import * as React from 'react'; import { FaLightbulb, FaTractor } from 'react-icons/fa'; import { Menu, MenuItem, ProSidebar, SubMenu } from 'react-pro-sidebar'; -import TuneRoundedIcon from '@mui/icons-material/TuneRounded'; -import WidgetsRoundedIcon from '@mui/icons-material/WidgetsRounded'; -import CameraAltRoundedIcon from '@mui/icons-material/CameraAltRounded'; -import ReceiptLongRoundedIcon from '@mui/icons-material/ReceiptLongRounded'; - import Box from '@mui/material/Box'; +import CameraAltRoundedIcon from '@mui/icons-material/CameraAltRounded'; +import Divider from '@mui/material/Divider'; import { Leva } from 'leva'; +import ReceiptLongRoundedIcon from '@mui/icons-material/ReceiptLongRounded'; import Tab from '@mui/material/Tab'; import Tabs from '@mui/material/Tabs'; +import TuneRoundedIcon from '@mui/icons-material/TuneRounded'; import Typography from '@mui/material/Typography'; -import Divider from '@mui/material/Divider'; +import WidgetsRoundedIcon from '@mui/icons-material/WidgetsRounded'; import StatusPanel from './StatusPanel'; import SceneNode from '../../SceneNode'; import LevaTheme from '../ConfigPanel/leva_theme.json'; import CameraPanel from './CameraPanel'; +import { RenderControls } from '../ConfigPanel/ConfigPanel'; interface TabPanelProps { children: React.ReactNode; @@ -159,6 +159,7 @@ export function BasicTabs(props: BasicTabsProps) {
+ - + diff --git a/nerfactory/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx b/nerfactory/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx index f74210552..b929571e7 100644 --- a/nerfactory/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx +++ b/nerfactory/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx @@ -1,16 +1,18 @@ import * as THREE from 'three'; import React, { useContext, useEffect, useRef } from 'react'; +import Select, { SelectChangeEvent } from '@mui/material/Select'; +import { useDispatch, useSelector } from 'react-redux'; +import FormControl from '@mui/material/FormControl'; import { IconButton } from '@mui/material'; +import InputLabel from '@mui/material/InputLabel'; +import MenuItem from '@mui/material/MenuItem'; import OpenWithIcon from '@mui/icons-material/OpenWith'; -import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls'; import PublicOffSharpIcon from '@mui/icons-material/PublicOffSharp'; import PublicSharpIcon from '@mui/icons-material/PublicSharp'; import Stats from 'stats.js'; import SyncOutlinedIcon from '@mui/icons-material/SyncOutlined'; -import { TransformControls } from 'three/examples/jsm/controls/TransformControls'; -import { useSelector } from 'react-redux'; import WebRtcWindow from '../WebRtcWindow/WebRtcWindow'; import { WebSocketContext } from '../WebSocket/WebSocket'; @@ -25,6 +27,39 @@ function createStats() { return stats; } +function CameraDropdown() { + const dispatch = useDispatch(); + const camera_options = useSelector( + (state) => state.renderingState.camera_options, + ); + const camera_choice = useSelector( + (state) => state.renderingState.camera_choice, + ); + const set_camera_choice = (event: SelectChangeEvent) => { + const value = event.target.value; + dispatch({ + type: 'write', + path: 'renderingState/camera_choice', + data: value, + }); + }; + + const menu_items = camera_options.map((camera_option) => ( + + {camera_option} + + )); + + return ( + + Camera + + + ); +} + function TransformIcons(props) { const sceneTree = props.sceneTree; const transform_controls = sceneTree.find_object(['Transform Controls']); @@ -71,59 +106,90 @@ function TransformIcons(props) { // manages a camera and the web rtc stream... export default function ViewerWindow(props) { - // eslint-disable-next-line react/prop-types const sceneTree = props.sceneTree; - const scene = props.scene; + const scene = sceneTree.object; + const renderer = sceneTree.metadata.renderer; const myRef = useRef(null); const websocket = useContext(WebSocketContext).socket; const field_of_view = useSelector( (state) => state.renderingState.field_of_view, ); - const field_of_view_ref = useRef(field_of_view); - let cameraControls = null; - let transformsControls = null; - let renderer = null; - let viewportWidth = null; - let viewportHeight = null; + const camera_choice = useSelector( + (state) => state.renderingState.camera_choice, + ); + + // listen to the viewport width + const size = new THREE.Vector2(); + renderer.getSize(size); + const [viewport_size, setDimensions] = React.useState({ + height: size.x, + width: size.y, + }); + const viewport_width = viewport_size.width; + const viewport_height = viewport_size.height; + + // on change, update the camera and controls + sceneTree.metadata.camera = sceneTree.find_object(['Cameras', camera_choice]); + let stats = null; - const camera = sceneTree.find(['Main Camera', '']).object; - - const getViewportWidth = () => { + const get_window_width = () => { const width = myRef.current.clientWidth; return width - (width % 2); }; - const getViewportHeight = () => { + const get_window_height = () => { return myRef.current.clientHeight; }; const handleResize = () => { - viewportWidth = getViewportWidth(); - viewportHeight = getViewportHeight(); - camera.aspect = viewportWidth / viewportHeight; - camera.updateProjectionMatrix(); + const viewportWidth = get_window_width(); + const viewportHeight = get_window_height(); + sceneTree.metadata.camera.aspect = viewportWidth / viewportHeight; + sceneTree.metadata.camera.updateProjectionMatrix(); renderer.setSize(viewportWidth, viewportHeight); }; + // update the camera information in the python server const sendCamera = () => { - // update the camera information in the python server if (websocket.readyState === WebSocket.OPEN) { const cmd = 'write'; const path = 'renderingState/camera'; const data = { type: cmd, path, - data: camera.toJSON(), + data: sceneTree.metadata.camera.toJSON(), }; const message = msgpack.encode(data); websocket.send(message); } }; + useEffect(() => { + function update_dimensions() { + setDimensions({ + height: get_window_height(), + width: get_window_width(), + }); + } + + window.addEventListener('resize', update_dimensions); + + // force a rerender + setDimensions({ + height: get_window_height(), + width: get_window_width(), + }); + + return () => { + window.removeEventListener('resize', update_dimensions); + }; + }, []); + // keep sending the camera often + // rerun this when the websocket changes useEffect(() => { const fps = 24; const interval = 1000 / fps; @@ -133,78 +199,89 @@ export default function ViewerWindow(props) { }; }, [websocket]); - const update = () => { - requestAnimationFrame(update); + const render = () => { + requestAnimationFrame(render); handleResize(); - camera.fov = field_of_view_ref.current; - camera.updateProjectionMatrix(); - cameraControls.update(); - renderer.render(scene, camera); + sceneTree.metadata.camera.updateProjectionMatrix(); + sceneTree.metadata.camera_controls.update(); + renderer.render(scene, sceneTree.metadata.camera); stats.update(); }; - // this is run once + // start the three.js rendering loop + // when the DOM is ready useEffect(() => { - viewportWidth = getViewportWidth(); - viewportHeight = getViewportHeight(); - stats = createStats(); myRef.current.append(stats.domElement); - - // camera = new THREE.PerspectiveCamera( - // field_of_view_ref.current, - // viewportWidth / viewportHeight, - // 0.01, - // 100, - // ); - camera.position.x = 5; - camera.position.y = -5; - camera.position.z = 5; - camera.up = new THREE.Vector3(0, 0, 1); - - renderer = new THREE.WebGLRenderer({ - antialias: true, - alpha: true, - }); - renderer.setPixelRatio(window.devicePixelRatio); - renderer.setSize(viewportWidth, viewportHeight); - myRef.current.append(renderer.domElement); - // add camera controls - cameraControls = new OrbitControls(camera, renderer.domElement); - cameraControls.rotateSpeed = 2.0; - cameraControls.zoomSpeed = 0.3; - cameraControls.panSpeed = 0.2; - cameraControls.target.set(0, 0, 0); // focus point of the controls - cameraControls.autoRotate = false; - cameraControls.enableDamping = true; - cameraControls.dampingFactor = 1.0; - cameraControls.update(); - - transformsControls = new TransformControls(camera, renderer.domElement); - sceneTree.set_object_from_path(['Transform Controls'], transformsControls); - - transformsControls.addEventListener('dragging-changed', (event) => { - cameraControls.enabled = !event.value; - }); - - update(); - // cameraControls.addEventListener('change', update); + render(); }, []); - // updates the field of view inside the ref to avoid rerendering so often + // updates the current camera with the field of view useEffect(() => { - field_of_view_ref.current = field_of_view; + sceneTree.metadata.camera.fov = field_of_view; }, [field_of_view]); + const render_height = useSelector( + (state) => state.renderingState.render_height, + ); + const render_width = useSelector( + (state) => state.renderingState.render_width, + ); + + let crop_w; + let crop_h; + const render_aspect = render_width / render_height; + const viewport_aspect = viewport_width / viewport_height; + let render_viewport_apsect_ratio = null; + if (render_aspect > viewport_aspect) { + // render width is the limiting factor + crop_w = viewport_width; + crop_h = viewport_width / render_aspect; + render_viewport_apsect_ratio = viewport_aspect / render_aspect; + } else { + // render height is the limiting factor + crop_w = viewport_height * render_aspect; + crop_h = viewport_height; + render_viewport_apsect_ratio = 1.0; + } + + let display = null; + if (camera_choice === 'Main Camera') { + display = 'none'; + } else { + display = 'flex'; + } + + const crop_style = { + display, + width: crop_w, + height: crop_h, + }; + + // set the threejs field of view + // such that the rendered video will match correctly + if (camera_choice !== 'Main Camera') { + const fl = 1.0 / Math.tan(field_of_view * Math.PI / 360); + const fl_new = fl * render_viewport_apsect_ratio; + const fov = Math.atan(1/fl_new) / (Math.PI / 360); + sceneTree.metadata.camera.fov = fov; + } + return ( <> {/* the webrtc viewer needs to know the camera pose */}
+
+ +
+
+
+
); } diff --git a/nerfactory/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx b/nerfactory/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx index 55398e4cd..2fe831c3d 100644 --- a/nerfactory/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx +++ b/nerfactory/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx @@ -40,7 +40,6 @@ export default function WebRtcWindow() { // { // urls: 'stun:stun.l.google.com:19302', // }, - // extra TURN server(s): // { // urls: 'turn:1830walnut.ddns.net', // username: 'turnuser', diff --git a/nerfactory/viewer/app/src/modules/WebSocket/WebSocket.jsx b/nerfactory/viewer/app/src/modules/WebSocket/WebSocket.jsx index 44e74675c..505161631 100644 --- a/nerfactory/viewer/app/src/modules/WebSocket/WebSocket.jsx +++ b/nerfactory/viewer/app/src/modules/WebSocket/WebSocket.jsx @@ -19,7 +19,6 @@ export default function WebSocketContextFunction({ children }) { const websocket_url = useSelector( (state) => state.websocketState.websocket_url, ); - console.log(websocket_url); const connect = () => { const url = `ws://${websocket_url}/`; diff --git a/nerfactory/viewer/app/src/reducer.js b/nerfactory/viewer/app/src/reducer.js index a77390600..62db036bf 100644 --- a/nerfactory/viewer/app/src/reducer.js +++ b/nerfactory/viewer/app/src/reducer.js @@ -10,9 +10,19 @@ const initialState = { webrtcState: { isConnected: false, }, + // for sending actual commands to the client + camera_path_payload: null, // the rendering state renderingState: { - camera: null, // the camera object used to render the three.js scene + // cameras + camera_options: ['Main Camera', 'Render Camera'], // the possible camera names to render the scene + camera_choice: 'Main Camera', // the camera being used to render the scene + + // camera path information + config_base_dir: 'config_base_dir', // the base directory of the config file + render_height: 1080, + render_width: 1920, + isTraining: true, output_options: ['rgb'], // populated by the possible Graph outputs output_choice: 'rgb', // the selected output @@ -25,7 +35,6 @@ const initialState = { train_eta: 'Paused', vis_train_ratio: 'Paused', }, - // the scene state sceneState: { sceneBounds: null, diff --git a/nerfactory/viewer/server/utils.py b/nerfactory/viewer/server/utils.py index e13fc91c1..988c27f3a 100644 --- a/nerfactory/viewer/server/utils.py +++ b/nerfactory/viewer/server/utils.py @@ -43,10 +43,23 @@ def get_chunks( return chunks +def three_js_perspective_camera_focal_length(fov: float, image_height: int): + """Returns the focal length of a three.js perspective camera. + + Args: + fov: the field of view of the camera in degrees. + image_height: the height of the image in pixels. + """ + pp_h = image_height / 2.0 + focal_length = pp_h / np.tan(fov * (np.pi / 180.0) / 2.0) + return focal_length + + def get_intrinsics_matrix_and_camera_to_world_h( camera_object: Dict[str, Any], image_height: int ) -> Tuple[torch.Tensor, torch.Tensor]: """Returns the camera intrinsics matrix and the camera to world homogeneous matrix. + Args: camera_object: a Camera object. image_size: the size of the image (height, width) @@ -57,7 +70,7 @@ def get_intrinsics_matrix_and_camera_to_world_h( image_width = aspect * image_height pp_w = image_width / 2.0 pp_h = image_height / 2.0 - focal_length = pp_h / np.tan(fov * (np.pi / 180.0) / 2.0) + focal_length = three_js_perspective_camera_focal_length(fov, image_height) intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float() # extrinsics diff --git a/nerfactory/viewer/server/viewer_utils.py b/nerfactory/viewer/server/viewer_utils.py index 8e23704c9..0a7904681 100644 --- a/nerfactory/viewer/server/viewer_utils.py +++ b/nerfactory/viewer/server/viewer_utils.py @@ -17,8 +17,10 @@ import enum import logging +import os import sys import threading +from pathlib import Path from typing import Any, Dict, Optional import numpy as np @@ -32,6 +34,7 @@ from nerfactory.datamanagers.datasets import InputDataset from nerfactory.models.base import Model from nerfactory.utils import profiler, visualization, writer from nerfactory.utils.decorators import check_visualizer_enabled, decorate_all +from nerfactory.utils.io import load_from_json, write_to_json from nerfactory.utils.misc import get_dict_to_torch from nerfactory.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter from nerfactory.viewer.server.subprocess import run_viewer_bridge_server_as_subprocess @@ -185,8 +188,9 @@ class VisualizerState: config: viewer setup configuration """ - def __init__(self, config: cfg.ViewerConfig): + def __init__(self, config: cfg.ViewerConfig, config_base_dir: Optional[Path] = None): self.config = config + self.config_base_dir = config_base_dir self.vis = None self.viewer_url = None @@ -203,7 +207,9 @@ class VisualizerState: # of the logging stack and easy to see and click # TODO(ethan): log the output of the viewer bridge server in a file where the training logs go console.line() - self.viewer_url = f"https://viewer.nerfactory.com/latest/?websocket_url=localhost:{websocket_port}" + json_filename = os.path.join(os.path.dirname(__file__), "../app/package.json") + version = load_from_json(Path(json_filename))["version"] + self.viewer_url = f"https://viewer.nerfactory.com/{version}/?websocket_url=localhost:{websocket_port}" viewer_url_local = f"http://localhost:4000/?websocket_url=localhost:{websocket_port}" pub_open_viewer_instructions_string = f"[Public] Open the viewer at {self.viewer_url}" dev_open_viewer_instructions_string = f"[Local] Open the viewer at {viewer_url_local}" @@ -240,6 +246,10 @@ class VisualizerState: start_train: whether to start train when viewer init; if False, only displays dataset until resume train is toggled """ + # set the config base dir + if self.config_base_dir: + self.vis["renderingState/config_base_dir"].write(str(self.config_base_dir)) + # clear the current scene self.vis["sceneState/sceneBounds"].delete() self.vis["sceneState/cameras"].delete() @@ -277,6 +287,15 @@ class VisualizerState: is_training = self.vis["renderingState/isTraining"].read() self.step = step + # check if we should interrupt from a button press? + camera_path_payload = self.vis["camera_path_payload"].read() + if camera_path_payload: + # write to json file + camera_path_filename = camera_path_payload["camera_path_filename"] + camera_path = camera_path_payload["camera_path"] + write_to_json(Path(camera_path_filename), camera_path) + self.vis["camera_path_payload"].delete() + camera_object = self._get_camera_object() if camera_object is None: return diff --git a/pyproject.toml b/pyproject.toml index 26ea8dd3c..b4a78a17a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ docs = [ ] [project.scripts] -viewer-bridge-server = "nerfactory.viewer.server.server:run_viewer_bridge_server" +nerfactory-viewer-bridge-server = "nerfactory.viewer.server.server:run_viewer_bridge_server" [options] # equivalent to using --extra-index-url with pip, which is needed for specifying the CUDA version torch and torchvision diff --git a/scripts/run_eval.py b/scripts/run_eval.py index 498b47d20..fe1854a78 100644 --- a/scripts/run_eval.py +++ b/scripts/run_eval.py @@ -18,8 +18,10 @@ import yaml from tqdm import tqdm from typing_extensions import assert_never +# pylint: disable=unused-import from nerfactory.cameras.camera_paths import ( get_interpolated_camera_path, + get_path_from_json, get_spiral_path, ) from nerfactory.cameras.cameras import Cameras @@ -98,6 +100,7 @@ def _render_trajectory_video( rendered_output_name: str, rendered_resolution_scaling_factor: float = 1.0, num_rays_per_chunk: int = 4096, + seconds: float = 5.0, ) -> None: """Helper function to create a video of the spiral trajectory. @@ -109,6 +112,7 @@ def _render_trajectory_video( rendered_resolution_scaling_factor: Scaling factor to apply to the camera image resolution. Defaults to 1.0. num_rays_per_chunk: Number of rays to use per chunk. Defaults to 4096. + seconds: Number for the output video. Defaults to 5.0. """ print("Creating trajectory video.") images = [] @@ -121,7 +125,6 @@ def _render_trajectory_video( image = outputs[rendered_output_name].cpu().numpy() images.append(image) - seconds = 5.0 fps = len(images) / seconds media.write_video(output_filename, images, fps=fps) @@ -198,24 +201,37 @@ class RenderTrajectory: # Name of the renderer output to use. rgb, depth, etc. rendered_output_name: str = "rgb" # Trajectory to render. - traj: Literal["spiral", "interp"] = "spiral" + traj: Literal["spiral", "interp", "filename"] = "spiral" # Scaling factor to apply to the camera image resolution. - rendered_resolution_scaling_factor: float = 1.0 + downscale_factor: int = 1 + # Filename of the camera path to render. + camera_path_filename: Path = Path("camera_path.json") # Name of the output file. output_path: Path = Path("output.mp4") + # How long the video should be. + seconds: float = 5.0 def main(self) -> None: """Main function.""" config, pipeline, _ = _eval_setup(self.load_config) + seconds = self.seconds + # TODO(ethan): use camera information from parsing args if self.traj == "spiral": camera_start = pipeline.datamanager.eval_dataloader.get_camera(image_idx=0) # TODO(ethan): pass in the up direction of the camera camera_path = get_spiral_path(camera_start, steps=30, radius=0.1) elif self.traj == "interp": - cameras = pipeline.datamanager.eval_dataloader.get_camera(image_idx=[0, 10]) - camera_path = get_interpolated_camera_path(cameras, steps=30) + # cameras_a = pipeline.datamanager.eval_dataloader.get_camera(image_idx=0) + # cameras_b = pipeline.datamanager.eval_dataloader.get_camera(image_idx=10) + # camera_path = get_interpolated_camera_path(cameras, steps=30) + raise NotImplementedError("Interpolated camera path not implemented.") + elif self.traj == "filename": + with open(self.camera_path_filename, "r", encoding="utf-8") as f: + camera_path = json.load(f) + seconds = camera_path["seconds"] + camera_path = get_path_from_json(camera_path) else: assert_never(self.traj) @@ -224,8 +240,9 @@ class RenderTrajectory: camera_path, output_filename=self.output_path, rendered_output_name=self.rendered_output_name, - rendered_resolution_scaling_factor=self.rendered_resolution_scaling_factor, + rendered_resolution_scaling_factor=1.0 / self.downscale_factor, num_rays_per_chunk=config.pipeline.datamanager.eval_num_rays_per_chunk, + seconds=seconds, )