From 310b177406e315b2983e8a5bfed059a69af01d56 Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Wed, 19 Apr 2023 00:18:17 +0000 Subject: [PATCH] remove visibility when rendering if not a target. also colors for train vs. eval images --- .../data/dataparsers/nerfstudio_dataparser.py | 36 ++++++++++++------- .../viewer/app/src/modules/Scene/Scene.jsx | 4 +-- .../viewer/app/src/modules/Scene/drawing.js | 18 ++++------ scripts/render.py | 4 +++ 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index f5b528369..3a7e3274f 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -61,7 +61,7 @@ class NerfstudioDataParserConfig(DataParserConfig): """The method to use to center the poses.""" auto_scale_poses: bool = True """Whether to automatically scale the poses to fit in +/- 1 bounding box.""" - eval_mode: Literal["train-split-fraction", "eval-frame-index"] = "train-split-fraction" + eval_mode: Literal["train-split-fraction", "eval-frame-index", "eval-interval"] = "train-split-fraction" """The method to use for splitting the dataset into train and eval.""" train_split_fraction: float = 0.9 """The fraction of images to use for training. The remaining images are for eval.""" @@ -69,6 +69,8 @@ class NerfstudioDataParserConfig(DataParserConfig): """The index of the frames to use for train.""" eval_frame_indices: Tuple[int, ...] = (1,) """The index of the frames to use for eval.""" + eval_interval: int = 8 + """The interval between frames to use for eval.""" depth_unit_scale_factor: float = 1e-3 """Scales the depth values to meters. Default value is 0.001 for a millimeter to meter conversion.""" @@ -213,6 +215,8 @@ class Nerfstudio(DataParser): indices = [i for i, path in enumerate(image_filenames) if path in split_filenames] CONSOLE.log(f"[yellow] Dataset is overriding {split}_indices to {indices}") indices = np.array(indices, dtype=np.int32) + i_train = indices + i_eval = indices elif has_split_files_spec: raise RuntimeError(f"The dataset's list of filenames for split {split} is missing.") else: @@ -226,12 +230,6 @@ class Nerfstudio(DataParser): ) # equally spaced training images starting and ending at 0 and num_images-1 i_eval = np.setdiff1d(i_all, i_train) # eval images are the remaining images assert len(i_eval) == num_eval_images - if split == "train": - indices = i_train - elif split in ["val", "test"]: - indices = i_eval - else: - raise ValueError(f"Unknown dataparser split {split}") elif self.config.eval_mode == "eval-frame-index": # keep around some metadata eval_frame_index_0_metadata = [] @@ -256,12 +254,24 @@ class Nerfstudio(DataParser): eval_frame_index_0_metadata.append(0) if frame_index in self.config.eval_frame_indices: i_eval.append(idx) - if split == "train": - indices = i_train - elif split in ["val", "test"]: - indices = i_eval - else: - raise ValueError(f"Unknown dataparser split {split}") + elif self.config.eval_mode == "eval-interval": + # filter image_filenames and poses based on a specified interval for eval + # this chunk of code is very similar to the mipnerf360 code + num_images = len(image_filenames) + all_indices = np.arange(num_images) + train_indices = all_indices[all_indices % self.config.eval_interval != 0] + eval_indices = all_indices[all_indices % self.config.eval_interval == 0] + i_train = train_indices + i_eval = eval_indices + else: + raise ValueError(f"Unknown eval mode {self.config.eval_mode}") + + if split == "train": + indices = i_train + elif split in ["val", "test"]: + indices = i_eval + else: + raise ValueError(f"Unknown dataparser split {split}") if "orientation_override" in meta: orientation_method = meta["orientation_override"] diff --git a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx index ef7b64e35..f645cd86d 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx +++ b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx @@ -271,7 +271,7 @@ export function get_scene_tree() { if (!prev.has(key)) { // keys_valid.push(key); const json = current[key]; - const camera = drawCamera(json, key); + const camera = drawCamera(json, key, 0x000000, 0xffffff); sceneTree.set_object_from_path([CAMERAS_NAME, key], camera); } } @@ -308,7 +308,7 @@ export function get_scene_tree() { if (!prev.has(key)) { // keys_valid.push(key); const json = current[key]; - const camera = drawCamera(json, key); + const camera = drawCamera(json, key, 0xff0000, 0xff5555); sceneTree.set_object_from_path([EVAL_CAMERAS_NAME, key], camera); } } diff --git a/nerfstudio/viewer/app/src/modules/Scene/drawing.js b/nerfstudio/viewer/app/src/modules/Scene/drawing.js index fb5ef7af5..e4d8d2292 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/drawing.js +++ b/nerfstudio/viewer/app/src/modules/Scene/drawing.js @@ -50,6 +50,7 @@ export function getCameraWireframe( focalLength = 4, w = 1.5, h = 2, + color = 0x000000, ) { // Returns a wireframe of a 3D line-plot of a camera symbol. // A wireframe is a frustum. @@ -93,17 +94,18 @@ export function getCameraWireframe( const geometry = new THREE.BufferGeometry().setFromPoints(points); const material = new THREE.LineBasicMaterial({ - color: 0x000000, + color: color, linewidth: 1, }); const lines = new THREE.LineSegments(geometry, material); return lines; } -export function drawCameraImagePlane(width, height, imageString, name) { +export function drawCameraImagePlane(width, height, imageString, name, color = 0xffffff) { // imageString is the texture as a base64 string const geometry = new THREE.PlaneGeometry(width, height); const material = new THREE.MeshBasicMaterial({ + color: color, side: THREE.DoubleSide, }); const texture = new THREE.TextureLoader().load(imageString); @@ -117,7 +119,7 @@ function transpose(matrix) { return matrix[0].map((col, i) => matrix.map((row) => row[i])); } -export function drawCamera(camera, name): THREE.Object3D { +export function drawCamera(camera, name, wireframe_color = 0x000000, image_plane_color = 0xffffff): THREE.Object3D { const group = new THREE.Group(); console.assert( @@ -133,6 +135,7 @@ export function drawCamera(camera, name): THREE.Object3D { displayedFocalLength, width, height, + wireframe_color, ); cameraWireframeObject.translateZ(displayedFocalLength); // move the wireframe frustum back group.add(cameraWireframeObject); @@ -141,6 +144,7 @@ export function drawCamera(camera, name): THREE.Object3D { height * 2, camera.image, name, + image_plane_color ); group.add(cameraImagePlaneObject); @@ -156,11 +160,3 @@ export function drawCamera(camera, name): THREE.Object3D { return group; } - -export function drawCameras(cameras): Record { - const cameraObjects = {}; - for (const [key, camera] of Object.entries(cameras)) { - cameraObjects[key] = drawCamera(camera); - } - return cameraObjects; -} diff --git a/scripts/render.py b/scripts/render.py index 453491639..4188b9dae 100644 --- a/scripts/render.py +++ b/scripts/render.py @@ -376,6 +376,10 @@ class RenderTrajectory: else: assert_never(self.traj) + # check if visibility is a desired output. if not, disable it for fast rendering + if "visibility" not in self.rendered_output_names: + pipeline.model.visibility_field = None + _render_trajectory_video( pipeline, camera_path,