From 4110bd7a3e55b786df6773356252048c5e5858f9 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sat, 1 Oct 2022 20:41:36 -0700 Subject: [PATCH] Rename scene bounds (#605) --- docs/tutorials/data/custom_dataset_format.md | 2 +- docs/tutorials/viewer/viewer_notebook.ipynb | 20 ++++++++--------- .../data/dataparsers/base_dataparser.py | 6 ++--- .../data/dataparsers/blender_dataparser.py | 6 ++--- .../data/dataparsers/friends_dataparser.py | 16 +++++++------- .../dataparsers/instant_ngp_dataparser.py | 6 ++--- .../data/dataparsers/mipnerf_dataparser.py | 6 ++--- .../data/dataparsers/nerfstudio_dataparser.py | 6 ++--- .../data/dataparsers/record3d_dataparser.py | 7 +++--- nerfstudio/data/scene_box.py | 22 +++++++++---------- nerfstudio/fields/density_fields.py | 4 ++-- nerfstudio/fields/instant_ngp_field.py | 4 ++-- nerfstudio/fields/nerfacto_field.py | 6 ++--- .../model_components/scene_colliders.py | 20 ++++++++--------- nerfstudio/models/base.py | 8 +++---- nerfstudio/models/instant_ngp.py | 6 ++--- nerfstudio/models/nerfacto.py | 6 ++--- nerfstudio/models/nerfw.py | 2 +- nerfstudio/models/semantic_nerf.py | 2 +- nerfstudio/pipelines/base.py | 4 ++-- nerfstudio/viewer/app/package.json | 2 +- .../viewer/app/src/modules/Scene/Scene.jsx | 20 ++++++++--------- .../viewer/app/src/modules/Scene/drawing.js | 9 +++----- nerfstudio/viewer/app/src/reducer.js | 2 +- nerfstudio/viewer/server/viewer_utils.py | 8 +++---- 25 files changed, 98 insertions(+), 102 deletions(-) diff --git a/docs/tutorials/data/custom_dataset_format.md b/docs/tutorials/data/custom_dataset_format.md index bdf38a529..271914fbc 100644 --- a/docs/tutorials/data/custom_dataset_format.md +++ b/docs/tutorials/data/custom_dataset_format.md @@ -16,7 +16,7 @@ class DatasetInputs: camera_to_world: torch.tensor = None mask_filenames: List[str] = None depth_filenames: List[str] = None - scene_bounds: SceneBounds = SceneBounds() + scene_box: SceneBox = SceneBox() semantics: Semantics = Semantics() point_cloud: PointCloud = PointCloud() alpha_color: Optional[TensorType[3]] = None diff --git a/docs/tutorials/viewer/viewer_notebook.ipynb b/docs/tutorials/viewer/viewer_notebook.ipynb index 8e9b4e441..5e74dd7da 100644 --- a/docs/tutorials/viewer/viewer_notebook.ipynb +++ b/docs/tutorials/viewer/viewer_notebook.ipynb @@ -38,7 +38,7 @@ "import numpy as np\n", "import torch\n", "import mediapy as media\n", - "from nerfstudio.data.structs import SceneBounds\n", + "from nerfstudio.data.scene_box import SceneBox\n", "from nerfstudio.cameras.cameras import PinholeCamera\n", "import time" ] @@ -142,15 +142,15 @@ } ], "source": [ - "# draw scene bounds in the viewer\n", + "# draw scene box in the viewer\n", "s = 1\n", - "scene_bounds = SceneBounds(aabb=torch.tensor([[-s, -s, -s], [s, s, s]]))\n", - "scene_bounds_json = scene_bounds.to_json()\n", + "scene_box = SceneBox(aabb=torch.tensor([[-s, -s, -s], [s, s, s]]))\n", + "scene_box_json = scene_box.to_json()\n", "\n", - "# draw scene bounds in the viewer\n", - "vis[\"sceneState/sceneBounds\"].write(scene_bounds_json)\n", + "# draw scene box in the viewer\n", + "vis[\"sceneState/sceneBox\"].write(scene_box_json)\n", "\n", - "# draw scene bounds in plotly\n", + "# draw scene box in plotly\n", "import plotly.graph_objects as go\n", "\n", "# TODO:" @@ -494,7 +494,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3.8.13 ('nerfactory')", + "display_name": "Python 3.8.12 ('nerfactory')", "language": "python", "name": "python3" }, @@ -508,11 +508,11 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.8.12" }, "vscode": { "interpreter": { - "hash": "100252972d4ee5286ab455ef1c778abde05b4affca9d1be78633d3797abf7e8c" + "hash": "34c28001ff35fb390494047002768a8182dcf55b1b11415165e62ea61557ab83" } } }, diff --git a/nerfstudio/data/dataparsers/base_dataparser.py b/nerfstudio/data/dataparsers/base_dataparser.py index b32796bc5..05dbd5e15 100644 --- a/nerfstudio/data/dataparsers/base_dataparser.py +++ b/nerfstudio/data/dataparsers/base_dataparser.py @@ -27,7 +27,7 @@ from torchtyping import TensorType import nerfstudio.configs.base_config as cfg from nerfstudio.cameras.cameras import Cameras from nerfstudio.configs.config_utils import to_immutable_dict -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox @dataclass @@ -58,8 +58,8 @@ class DataparserOutputs: """Camera object storing collection of camera information in dataset""" alpha_color: Optional[TensorType[3]] = None """color of dataset background""" - scene_bounds: SceneBounds = SceneBounds() - """scene bounds of dataset""" + scene_box: SceneBox = SceneBox() + """scene box of dataset. Could be used to bound the scene, or to provide the scene scale depending on model.""" # we support additional input information/formats including mask/depth/semantics. additional_inputs: Dict[str, Any] = to_immutable_dict({}) """Dictionary of additional dataset information (e.g. semantics/point clouds/masks). diff --git a/nerfstudio/data/dataparsers/blender_dataparser.py b/nerfstudio/data/dataparsers/blender_dataparser.py index 03c01aa53..1612e5086 100644 --- a/nerfstudio/data/dataparsers/blender_dataparser.py +++ b/nerfstudio/data/dataparsers/blender_dataparser.py @@ -29,7 +29,7 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataParserConfig, DataparserOutputs, ) -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.utils.colors import get_color from nerfstudio.utils.io import load_from_json @@ -88,7 +88,7 @@ class Blender(DataParser): # in x,y,z order camera_to_world[..., 3] *= self.scale_factor - scene_bounds = SceneBounds(aabb=torch.tensor([[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]], dtype=torch.float32)) + scene_box = SceneBox(aabb=torch.tensor([[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]], dtype=torch.float32)) cameras = Cameras( camera_to_worlds=camera_to_world, @@ -103,7 +103,7 @@ class Blender(DataParser): image_filenames=image_filenames, cameras=cameras, alpha_color=alpha_color_tensor, - scene_bounds=scene_bounds, + scene_box=scene_box, ) return dataset_inputs diff --git a/nerfstudio/data/dataparsers/friends_dataparser.py b/nerfstudio/data/dataparsers/friends_dataparser.py index f7ed972a7..1f4648712 100644 --- a/nerfstudio/data/dataparsers/friends_dataparser.py +++ b/nerfstudio/data/dataparsers/friends_dataparser.py @@ -31,7 +31,7 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataparserOutputs, Semantics, ) -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.utils.io import load_from_json CONSOLE = Console() @@ -122,18 +122,18 @@ class Friends(DataParser): scene_scale = self.config.scene_scale - # -- set the scene bounds --- - scene_bounds = SceneBounds(aabb=bbox) + # -- set the scene box --- + scene_box = SceneBox(aabb=bbox) # center the box and adjust the cameras too - center = scene_bounds.get_center() - scene_bounds.aabb -= center + center = scene_box.get_center() + scene_box.aabb -= center camera_to_worlds[..., 3] -= center # scale the longest dimension to match the cube size - lengths = scene_bounds.aabb[1] - scene_bounds.aabb[0] + lengths = scene_box.aabb[1] - scene_box.aabb[0] longest_dim = torch.argmax(lengths) longest_length = lengths[longest_dim] scale = scene_scale / longest_length - scene_bounds.aabb = scene_bounds.aabb * scale # box + scene_box.aabb = scene_box.aabb * scale # box camera_to_worlds[..., 3] *= scale # cameras # --- semantics --- @@ -185,7 +185,7 @@ class Friends(DataParser): dataset_inputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, - scene_bounds=scene_bounds, + scene_box=scene_box, additional_inputs={"semantics": {"func": get_semantics_and_masks, "kwargs": {"semantics": semantics}}}, ) return dataset_inputs diff --git a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py index 5ea545a5f..a6f2bf488 100644 --- a/nerfstudio/data/dataparsers/instant_ngp_dataparser.py +++ b/nerfstudio/data/dataparsers/instant_ngp_dataparser.py @@ -31,7 +31,7 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataParserConfig, DataparserOutputs, ) -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.utils.io import load_from_json @@ -88,7 +88,7 @@ class InstantNGP(DataParser): # in x,y,z order # assumes that the scene is centered at the origin aabb_scale = meta["aabb_scale"] - scene_bounds = SceneBounds( + scene_box = SceneBox( aabb=torch.tensor( [[-aabb_scale, -aabb_scale, -aabb_scale], [aabb_scale, aabb_scale, aabb_scale]], dtype=torch.float32 ) @@ -110,7 +110,7 @@ class InstantNGP(DataParser): dataset_inputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, - scene_bounds=scene_bounds, + scene_box=scene_box, ) return dataset_inputs diff --git a/nerfstudio/data/dataparsers/mipnerf_dataparser.py b/nerfstudio/data/dataparsers/mipnerf_dataparser.py index 305aca02c..4d85b4f22 100644 --- a/nerfstudio/data/dataparsers/mipnerf_dataparser.py +++ b/nerfstudio/data/dataparsers/mipnerf_dataparser.py @@ -29,7 +29,7 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataParserConfig, DataparserOutputs, ) -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox @dataclass @@ -142,7 +142,7 @@ class Mipnerf360(DataParser): intrinsics *= torch.tensor([cx, cy, focal_length]) aabb = torch.tensor([[-4, -4, -4], [4, 4, 4]], dtype=torch.float32) * self.config.aabb_scale - scene_bounds = SceneBounds(aabb=aabb) + scene_box = SceneBox(aabb=aabb) cameras = Cameras( fx=focal_length, @@ -158,7 +158,7 @@ class Mipnerf360(DataParser): dataset_inputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, - scene_bounds=scene_bounds, + scene_box=scene_box, ) return dataset_inputs diff --git a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py index 9ff7636db..3f445f36e 100644 --- a/nerfstudio/data/dataparsers/nerfstudio_dataparser.py +++ b/nerfstudio/data/dataparsers/nerfstudio_dataparser.py @@ -31,7 +31,7 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataParserConfig, DataparserOutputs, ) -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.utils.io import load_from_json @@ -123,7 +123,7 @@ class Nerfstudio(DataParser): # in x,y,z order # assumes that the scene is centered at the origin aabb_scale = self.config.scene_scale - scene_bounds = SceneBounds( + scene_box = SceneBox( aabb=torch.tensor( [[-aabb_scale, -aabb_scale, -aabb_scale], [aabb_scale, aabb_scale, aabb_scale]], dtype=torch.float32 ) @@ -160,6 +160,6 @@ class Nerfstudio(DataParser): dataset_inputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, - scene_bounds=scene_bounds, + scene_box=scene_box, ) return dataset_inputs diff --git a/nerfstudio/data/dataparsers/record3d_dataparser.py b/nerfstudio/data/dataparsers/record3d_dataparser.py index 21be800c2..1ef56ae11 100644 --- a/nerfstudio/data/dataparsers/record3d_dataparser.py +++ b/nerfstudio/data/dataparsers/record3d_dataparser.py @@ -30,7 +30,7 @@ from nerfstudio.data.dataparsers.base_dataparser import ( DataParserConfig, DataparserOutputs, ) -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.utils import poses as pose_utils from nerfstudio.utils.io import load_from_json @@ -132,9 +132,8 @@ class Record3D(DataParser): intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) intrinsics *= torch.tensor([cx, cy, focal_length]) - # scene_bounds = SceneBounds.from_camera_poses(camera_to_world, self.config.aabb_scale) aabb = torch.tensor([[-1, -1, -1], [1, 1, 1]], dtype=torch.float32) * self.config.aabb_scale - scene_bounds = SceneBounds(aabb=aabb) + scene_box = SceneBox(aabb=aabb) cameras = Cameras( fx=focal_length, @@ -148,7 +147,7 @@ class Record3D(DataParser): dataset_inputs = DataparserOutputs( image_filenames=image_filenames, cameras=cameras, - scene_bounds=scene_bounds, + scene_box=scene_box, ) return dataset_inputs diff --git a/nerfstudio/data/scene_box.py b/nerfstudio/data/scene_box.py index 9bf9319f7..acdb54ad6 100644 --- a/nerfstudio/data/scene_box.py +++ b/nerfstudio/data/scene_box.py @@ -24,8 +24,8 @@ from torchtyping import TensorType @dataclass -class SceneBounds: - """Data to represent the scene bounds.""" +class SceneBox: + """Data to represent the scene box.""" aabb: TensorType[2, 3] = None """aabb: axis-aligned bounding box. @@ -43,14 +43,14 @@ class SceneBounds: diff = self.aabb[1] - self.aabb[0] return self.aabb[0] + diff / 2.0 - def get_centered_and_scaled_scene_bounds(self, scale_factor: Union[float, torch.Tensor] = 1.0): + def get_centered_and_scaled_scene_box(self, scale_factor: Union[float, torch.Tensor] = 1.0): """Returns a new box that has been shifted and rescaled to be centered about the origin. Args: scale_factor: How much to scale the camera origins by. """ - return SceneBounds(aabb=(self.aabb - self.get_center()) * scale_factor) + return SceneBox(aabb=(self.aabb - self.get_center()) * scale_factor) @staticmethod def get_normalized_positions(positions: TensorType[..., 3], aabb: TensorType[2, 3]): @@ -69,19 +69,19 @@ class SceneBounds: return {"type": "aabb", "min_point": self.aabb[0].tolist(), "max_point": self.aabb[1].tolist()} @staticmethod - def from_json(json_: Dict) -> "SceneBounds": - """Returns the an instance of SceneBounds from a json dictionary. + def from_json(json_: Dict) -> "SceneBox": + """Returns the an instance of SceneBox from a json dictionary. Args: - json_: the json dictionary containing scene bound information + json_: the json dictionary containing scene box information """ assert json_["type"] == "aabb" aabb = torch.tensor([json_[0], json_[1]]) - return SceneBounds(aabb=aabb) + return SceneBox(aabb=aabb) @staticmethod - def from_camera_poses(poses: TensorType[..., 3, 4], scale_factor: float) -> "SceneBounds": - """Returns the instance of SceneBounds that fully envelopes a set of poses + def from_camera_poses(poses: TensorType[..., 3, 4], scale_factor: float) -> "SceneBox": + """Returns the instance of SceneBox that fully envelopes a set of poses Args: poses: tensor of camera pose matrices @@ -89,4 +89,4 @@ class SceneBounds: """ xyzs = poses[..., :3, -1] aabb = torch.stack([torch.min(xyzs, dim=0)[0], torch.max(xyzs, dim=0)[0]]) - return SceneBounds(aabb=aabb * scale_factor) + return SceneBox(aabb=aabb * scale_factor) diff --git a/nerfstudio/fields/density_fields.py b/nerfstudio/fields/density_fields.py index 06c150d44..e807213ac 100644 --- a/nerfstudio/fields/density_fields.py +++ b/nerfstudio/fields/density_fields.py @@ -25,7 +25,7 @@ from torch.nn.parameter import Parameter from torchtyping import TensorType from nerfstudio.cameras.rays import RaySamples -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.spatial_distortions import SpatialDistortion from nerfstudio.fields.base_field import Field from nerfstudio.utils.activations import trunc_exp @@ -103,7 +103,7 @@ class HashMLPDensityField(Field): positions = self.spatial_distortion(ray_samples.frustums.get_positions()) positions = (positions + 2.0) / 4.0 else: - positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) positions_flat = positions.view(-1, 3) # assert all positions are in the range [0, 1] # otherwise print min and max values diff --git a/nerfstudio/fields/instant_ngp_field.py b/nerfstudio/fields/instant_ngp_field.py index 9042051e6..dc0ec8c71 100644 --- a/nerfstudio/fields/instant_ngp_field.py +++ b/nerfstudio/fields/instant_ngp_field.py @@ -25,7 +25,7 @@ from torch.nn.parameter import Parameter from torchtyping import TensorType from nerfstudio.cameras.rays import RaySamples -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.embedding import Embedding from nerfstudio.field_components.field_heads import FieldHeadNames from nerfstudio.fields.base_field import Field @@ -154,7 +154,7 @@ class TCNNInstantNGPField(Field): d = self.direction_encoding(directions_flat) if density_embedding is None: - positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) h = torch.cat([d, positions.view(-1, 3)], dim=-1) else: h = torch.cat([d, density_embedding.view(-1, self.geo_feat_dim)], dim=-1) diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index 9082e2ad8..3a5cb2682 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -26,7 +26,7 @@ from torch.nn.parameter import Parameter from torchtyping import TensorType from nerfstudio.cameras.rays import RaySamples -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.field_components.embedding import Embedding from nerfstudio.field_components.encoding import Encoding, HashEncoding, SHEncoding from nerfstudio.field_components.field_heads import ( @@ -152,7 +152,7 @@ class TCNNNerfactoField(Field): positions = self.spatial_distortion(positions) positions = (positions + 2.0) / 4.0 else: - positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) positions_flat = positions.view(-1, 3) h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1) @@ -184,7 +184,7 @@ class TCNNNerfactoField(Field): ) if density_embedding is None: - positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + positions = SceneBox.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) h = torch.cat([d, positions.view(-1, 3), embedded_appearance], dim=-1) else: h = torch.cat( diff --git a/nerfstudio/model_components/scene_colliders.py b/nerfstudio/model_components/scene_colliders.py index 180970761..1398a4e25 100644 --- a/nerfstudio/model_components/scene_colliders.py +++ b/nerfstudio/model_components/scene_colliders.py @@ -23,10 +23,10 @@ from torch import nn from torchtyping import TensorType from nerfstudio.cameras.rays import RayBundle -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox -class SceneBoundsCollider(nn.Module): +class SceneBoxCollider(nn.Module): """Module for setting near and far values for rays.""" def __init__(self, **kwargs) -> None: @@ -38,16 +38,16 @@ class SceneBoundsCollider(nn.Module): raise NotImplementedError -class AABBBoxCollider(SceneBoundsCollider): - """Module for colliding rays with the scene bounds to compute near and far values. +class AABBBoxCollider(SceneBoxCollider): + """Module for colliding rays with the scene box to compute near and far values. Args: - scene_bounds: scene bounds to apply to dataset + scene_box: scene box to apply to dataset """ - def __init__(self, scene_bounds: SceneBounds, near_plane: float = 0.0, **kwargs) -> None: + def __init__(self, scene_box: SceneBox, near_plane: float = 0.0, **kwargs) -> None: super().__init__(**kwargs) - self.scene_bounds = scene_bounds + self.scene_box = scene_box self.near_plane = near_plane def _intersect_with_aabb( @@ -92,13 +92,13 @@ class AABBBoxCollider(SceneBoundsCollider): return nears, fars, valid_mask def forward(self, ray_bundle: RayBundle) -> RayBundle: - """Intersects the rays with the scene bounds and updates the near and far values. + """Intersects the rays with the scene box and updates the near and far values. Populates nears and fars fields and returns the ray_bundle. Args: ray_bundle: specified ray bundle to operate on """ - aabb = self.scene_bounds.aabb + aabb = self.scene_box.aabb nears, fars, valid_mask = self._intersect_with_aabb(ray_bundle.origins, ray_bundle.directions, aabb) ray_bundle.nears = nears[..., None] ray_bundle.fars = fars[..., None] @@ -106,7 +106,7 @@ class AABBBoxCollider(SceneBoundsCollider): return ray_bundle -class NearFarCollider(SceneBoundsCollider): +class NearFarCollider(SceneBoxCollider): """Sets the nears and fars with fixed values. Args: diff --git a/nerfstudio/models/base.py b/nerfstudio/models/base.py index 46a979656..b8341206b 100644 --- a/nerfstudio/models/base.py +++ b/nerfstudio/models/base.py @@ -30,7 +30,7 @@ from torch.nn import Parameter from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs.base_config import InstantiateConfig from nerfstudio.configs.config_utils import to_immutable_dict -from nerfstudio.data.scene_box import SceneBounds +from nerfstudio.data.scene_box import SceneBox from nerfstudio.model_components.scene_colliders import NearFarCollider from nerfstudio.utils.callbacks import TrainingCallback, TrainingCallbackAttributes @@ -59,7 +59,7 @@ class Model(nn.Module): Args: config: configuration for instantiating model - scene_bounds: dataset scene bounds + scene_box: dataset scene box """ config: ModelConfig @@ -67,13 +67,13 @@ class Model(nn.Module): def __init__( self, config: ModelConfig, - scene_bounds: SceneBounds, + scene_box: SceneBox, num_train_data: int, **kwargs, ) -> None: super().__init__() self.config = config - self.scene_bounds = scene_bounds + self.scene_box = scene_box self.num_train_data = num_train_data self.kwargs = kwargs self.collider = None diff --git a/nerfstudio/models/instant_ngp.py b/nerfstudio/models/instant_ngp.py index 153c54b86..9e31ffaad 100644 --- a/nerfstudio/models/instant_ngp.py +++ b/nerfstudio/models/instant_ngp.py @@ -98,13 +98,13 @@ class NGPModel(Model): super().populate_modules() self.field = TCNNInstantNGPField( - aabb=self.scene_bounds.aabb, + aabb=self.scene_box.aabb, contraction_type=self.config.contraction_type, use_appearance_embedding=self.config.use_appearance_embedding, num_images=self.num_train_data, ) - self.scene_aabb = Parameter(self.scene_bounds.aabb.flatten(), requires_grad=False) + self.scene_aabb = Parameter(self.scene_box.aabb.flatten(), requires_grad=False) # Occupancy Grid self.occupancy_grid = nerfacc.OccupancyGrid( @@ -114,7 +114,7 @@ class NGPModel(Model): ) # Sampler - vol_sampler_aabb = self.scene_bounds.aabb if self.config.contraction_type == ContractionType.AABB else None + vol_sampler_aabb = self.scene_box.aabb if self.config.contraction_type == ContractionType.AABB else None self.sampler = VolumetricSampler( scene_aabb=vol_sampler_aabb, occupancy_grid=self.occupancy_grid, diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index bd0cf2e49..43bde0238 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -105,7 +105,7 @@ class NerfactoModel(Model): # Fields if self.config.use_appearance_conditioning: self.field = TCNNNerfactoField( - self.scene_bounds.aabb, + self.scene_box.aabb, spatial_distortion=scene_contraction, num_images=self.num_train_data, use_average_appearance_embedding=self.config.use_average_appearance_embedding, @@ -116,12 +116,12 @@ class NerfactoModel(Model): # Build the proposal network(s) self.proposal_networks = torch.nn.ModuleList() if self.config.use_same_proposal_network: - network = HashMLPDensityField(self.scene_bounds.aabb, spatial_distortion=scene_contraction) + network = HashMLPDensityField(self.scene_box.aabb, spatial_distortion=scene_contraction) self.proposal_networks.append(network) self.density_fns = [network.density_fn for _ in range(self.config.num_proposal_network_iterations)] else: for _ in range(self.config.num_proposal_network_iterations): - network = HashMLPDensityField(self.scene_bounds.aabb, spatial_distortion=scene_contraction) + network = HashMLPDensityField(self.scene_box.aabb, spatial_distortion=scene_contraction) self.proposal_networks.append(network) self.density_fns = [network.density_fn for network in self.proposal_networks] diff --git a/nerfstudio/models/nerfw.py b/nerfstudio/models/nerfw.py index a2e2f5940..caa60c91b 100644 --- a/nerfstudio/models/nerfw.py +++ b/nerfstudio/models/nerfw.py @@ -139,7 +139,7 @@ class NerfWModel(Model): # colliders if self.config.enable_collider: - self.collider = AABBBoxCollider(scene_bounds=self.scene_bounds) + self.collider = AABBBoxCollider(scene_box=self.scene_box) def get_param_groups(self): param_groups = {} diff --git a/nerfstudio/models/semantic_nerf.py b/nerfstudio/models/semantic_nerf.py index 3bdccadec..d8c145b42 100644 --- a/nerfstudio/models/semantic_nerf.py +++ b/nerfstudio/models/semantic_nerf.py @@ -74,7 +74,7 @@ class SemanticNerfModel(NeRFModel): # colliders if self.config.enable_collider: - self.collider = AABBBoxCollider(scene_bounds=self.scene_bounds) + self.collider = AABBBoxCollider(scene_box=self.scene_box) def get_outputs(self, ray_bundle: RayBundle): # uniform sampling diff --git a/nerfstudio/pipelines/base.py b/nerfstudio/pipelines/base.py index 58b17c6fe..b7195c2eb 100644 --- a/nerfstudio/pipelines/base.py +++ b/nerfstudio/pipelines/base.py @@ -210,10 +210,10 @@ class VanillaPipeline(Pipeline): device=device, test_mode=test_mode, world_size=world_size, local_rank=local_rank ) self.datamanager.to(device) - # TODO(ethan): get rid of scene_bounds from the model + # TODO(ethan): get rid of scene_box from the model assert self.datamanager.train_input_dataset is not None, "Missing input dataset" self.model = config.model.setup( - scene_bounds=self.datamanager.train_input_dataset.dataset_inputs.scene_bounds, + scene_box=self.datamanager.train_input_dataset.dataset_inputs.scene_box, num_train_data=len(self.datamanager.train_input_dataset), ) self.model.to(device) diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index 59385f7f2..42b8445a1 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "22-09-30-0", + "version": "22-10-01-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx index 8723d795c..4ab87fd45 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx +++ b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx @@ -7,7 +7,7 @@ import { CSS2DRenderer } from 'three/examples/jsm/renderers/CSS2DRenderer'; import { TransformControls } from 'three/examples/jsm/controls/TransformControls'; import { useDispatch } from 'react-redux'; -import { drawCamera, drawSceneBounds } from './drawing'; +import { drawCamera, drawSceneBox } from './drawing'; import { CameraHelper } from '../SidePanel/CameraPanel/CameraHelper'; import SceneNode from '../../SceneNode'; @@ -16,7 +16,7 @@ import { subscribe_to_changes } from '../../subscriber'; const msgpack = require('msgpack-lite'); -const SCENE_BOUNDS_NAME = 'Scene Bounds'; +const SCENE_BOX_NAME = 'Scene Box'; const CAMERAS_NAME = 'Training Cameras'; export function get_scene_tree() { @@ -169,19 +169,19 @@ export function get_scene_tree() { const light = new THREE.AmbientLight(color, intensity); sceneTree.set_object_from_path(['Light'], light); - // draw scene bounds - const selector_fn_scene_bounds = (state) => { - return state.sceneState.sceneBounds; + // draw scene box + const selector_fn_scene_box = (state) => { + return state.sceneState.sceneBox; }; - const fn_value_scene_bounds = (previous, current) => { + const fn_value_scene_box = (previous, current) => { if (current !== null) { - const line = drawSceneBounds(current); - sceneTree.set_object_from_path([SCENE_BOUNDS_NAME], line); + const line = drawSceneBox(current); + sceneTree.set_object_from_path([SCENE_BOX_NAME], line); } else { - sceneTree.delete([SCENE_BOUNDS_NAME]); + sceneTree.delete([SCENE_BOX_NAME]); } }; - subscribe_to_changes(selector_fn_scene_bounds, fn_value_scene_bounds); + subscribe_to_changes(selector_fn_scene_box, fn_value_scene_box); // draw camera // NOTE: this has some issues right now! it won't diff --git a/nerfstudio/viewer/app/src/modules/Scene/drawing.js b/nerfstudio/viewer/app/src/modules/Scene/drawing.js index 12bf2f173..c2ae2996a 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/drawing.js +++ b/nerfstudio/viewer/app/src/modules/Scene/drawing.js @@ -2,13 +2,10 @@ // ---- code for drawing with three.js ---- import * as THREE from 'three'; -export function drawSceneBounds(sceneBounds): THREE.Object3D { - console.assert( - sceneBounds.type === 'aabb', - 'The scene bounds must be an AABB', - ); +export function drawSceneBox(sceneBox): THREE.Object3D { + console.assert(sceneBox.type === 'aabb', 'The scene box must be an AABB'); - const box = sceneBounds; + const box = sceneBox; const w = 1.0; const aaa = new THREE.Vector3(w, w, w); diff --git a/nerfstudio/viewer/app/src/reducer.js b/nerfstudio/viewer/app/src/reducer.js index ad91e028c..14566e299 100644 --- a/nerfstudio/viewer/app/src/reducer.js +++ b/nerfstudio/viewer/app/src/reducer.js @@ -37,7 +37,7 @@ const initialState = { }, // the scene state sceneState: { - sceneBounds: null, + sceneBox: null, cameras: null, }, // ------------------------------------------------------------------------ diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index ce67f609e..ebfd6506c 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -258,7 +258,7 @@ class ViewerState: self.vis["renderingState/config_base_dir"].write(str(self.log_filename.parents[0])) # clear the current scene - self.vis["sceneState/sceneBounds"].delete() + self.vis["sceneState/sceneBox"].delete() self.vis["sceneState/cameras"].delete() # draw the training cameras and images @@ -269,9 +269,9 @@ class ViewerState: camera_json = dataset.dataset_inputs.cameras.to_json(camera_idx=idx, image=bgr, max_size=100) self.vis[f"sceneState/cameras/{idx:06d}"].write(camera_json) - # draw the scene bounds (i.e., the bounding box) - json_ = dataset.dataset_inputs.scene_bounds.to_json() - self.vis["sceneState/sceneBounds"].write(json_) + # draw the scene box (i.e., the bounding box) + json_ = dataset.dataset_inputs.scene_box.to_json() + self.vis["sceneState/sceneBox"].write(json_) # set the initial state whether to train or not self.vis["renderingState/isTraining"].write(start_train)