diff --git a/Dockerfile b/Dockerfile
index f40c2404b..966db1cbc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -125,9 +125,8 @@ RUN CUDA_VER=${CUDA_VERSION%.*} && CUDA_VER=${CUDA_VER//./} && python3.10 -m pip
ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}
RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git@v1.6#subdirectory=bindings/torch
-# Install pycolmap 0.3.0, required by hloc.
-# TODO(https://github.com/colmap/pycolmap/issues/111) use wheel when available for Python 3.10
-RUN git clone --branch v0.3.0 --recursive https://github.com/colmap/pycolmap.git && \
+# Install pycolmap, required by hloc.
+RUN git clone --branch v0.4.0 --recursive https://github.com/colmap/pycolmap.git && \
cd pycolmap && \
python3.10 -m pip install . && \
cd ..
@@ -139,13 +138,13 @@ RUN git clone --branch master --recursive https://github.com/cvg/Hierarchical-Lo
cd ..
# Install pyceres from source
-RUN git clone --branch main --recursive https://github.com/cvg/pyceres.git && \
+RUN git clone --branch v1.0 --recursive https://github.com/cvg/pyceres.git && \
cd pyceres && \
python3.10 -m pip install -e . && \
cd ..
# Install pixel perfect sfm.
-RUN git clone --branch main --recursive https://github.com/cvg/pixel-perfect-sfm.git && \
+RUN git clone --branch v1.0 --recursive https://github.com/cvg/pixel-perfect-sfm.git && \
cd pixel-perfect-sfm && \
python3.10 -m pip install -e . && \
cd ..
diff --git a/docs/extensions/blender_addon.md b/docs/extensions/blender_addon.md
index 8e2fa4bd2..b433fd2cf 100644
--- a/docs/extensions/blender_addon.md
+++ b/docs/extensions/blender_addon.md
@@ -6,7 +6,7 @@
## Overview
-This Blender add-on allows for compositing with a Nerfstudio render as a background layer by generating a camera path JSON file from the Blender camera path, as well as a way to import Nerfstudio JSON files as a Blender camera baked with the Nerfstudio camera path. This add-on also allows compositing multiple NeRF objects into a NeRF scene. This is achieved by importing a mesh or point-cloud representation of the NeRF scene from Nerfstudio into Blender and getting the camera coordinates relative to the transformations of the NeRF representation. Dynamic FOV from the Blender camera is supported and will match the Nerfstudio render. Perspective and equirectangular cameras are supported.
+This Blender add-on allows for compositing with a Nerfstudio render as a background layer by generating a camera path JSON file from the Blender camera path, as well as a way to import Nerfstudio JSON files as a Blender camera baked with the Nerfstudio camera path. This add-on also allows compositing multiple NeRF objects into a NeRF scene. This is achieved by importing a mesh or point-cloud representation of the NeRF scene from Nerfstudio into Blender and getting the camera coordinates relative to the transformations of the NeRF representation. Dynamic FOV from the Blender camera is supported and will match the Nerfstudio render. Perspective, equirectangular, and omnidirectional stereo (VR 360) cameras are supported.
@@ -89,7 +89,7 @@ This Blender add-on allows for compositing with a Nerfstudio render as a backgro
- The generated JSON camera path follows the user specified frame start, end, and step fields in the Output Properties in Blender. The JSON file specifies the user specified x and y render resolutions at the given % in the Output Properties.
- The add-on computes the camera coordinates based on the active camera in the scene.
- FOV animated changes of the camera will be matched with the NeRF render.
-- Perspective and equirectangular cameras are supported and can be selected within Blender.
+- Perspective, equirectangular, and omnidirectional stereo cameras are supported and can be selected within Blender.
- The generated JSON file can be imported into Nerfstudio. Each keyframe of the camera transform in the frame sequence in Blender will be a keyframe in Nerfstudio. The exported JSON camera path is baked, where each frame in the sequence is a keyframe. This is to ensure that frame interpolation across Nerfstudio and Blender do not differ.
@@ -101,6 +101,10 @@ This Blender add-on allows for compositing with a Nerfstudio render as a backgro
- It is recommended to export a high fidelity mesh as the NeRF representation from Nerfstudio. However if that is not possible or the scene is too large, a point cloud representation also works.
- For compositing, it is recommended to convert the video render into image frames to ensure the Blender and NeRF renders are in sync.
- Currently, dynamic camera focus is not supported.
+- For compositing Blender objects or NeRFs with Nerfstudio omnidirectional stereo renders (VR 360), configure the Blender camera to be equirectangular and enable stereoscopy in the Output Properties. Under the Stereoscopy panel the Blender camera settings, change the mode to "Parallel", set the Interocular Distance to 0.064 m, and checkmark "Spherical Stereo".
+
+
+
- Fisheye and orthographic cameras are not supported.
- A walkthrough of this section is included in the tutorial video.
diff --git a/docs/nerfology/methods/imgs/tensorf/models_tensorf_factorization.png b/docs/nerfology/methods/imgs/tensorf/models_tensorf_factorization.png
new file mode 100644
index 000000000..8557a6c37
Binary files /dev/null and b/docs/nerfology/methods/imgs/tensorf/models_tensorf_factorization.png differ
diff --git a/docs/nerfology/methods/imgs/tensorf/models_tensorf_pipeline.png b/docs/nerfology/methods/imgs/tensorf/models_tensorf_pipeline.png
new file mode 100644
index 000000000..310e08652
Binary files /dev/null and b/docs/nerfology/methods/imgs/tensorf/models_tensorf_pipeline.png differ
diff --git a/docs/nerfology/methods/tensorf.md b/docs/nerfology/methods/tensorf.md
index d3b90786b..e418f5241 100644
--- a/docs/nerfology/methods/tensorf.md
+++ b/docs/nerfology/methods/tensorf.md
@@ -8,6 +8,10 @@
Paper Website
```
+
+
### Running Model
```bash
@@ -16,4 +20,14 @@ ns-train tensorf
## Overview
-🚧 Guide under construction 🚧
+```{image} imgs/tensorf/models_tensorf_pipeline.png
+:align: center
+```
+
+TensoRF models the radiance field of a scene as a 4D tensor, which represents a 3D voxel grid with per-voxel multi-channel features. TensoRF factorizes the 4D scene tensor into multiple compact low-rank tensor components using CP or VM modes. CP decomposition factorizes tensors into rank-one components with compact vectors. Vector-Matrix (VM) decomposition factorizes tensors into compact vector and matrix factors.
+
+```{image} imgs/tensorf/models_tensorf_factorization.png
+:align: center
+```
+
+TensoRF with CP(left) and VM(right) decompositions results in a significantly reduced memory footprint compared to previous and concurrent works that directly optimize per-voxel features, such as [Plenoxels](https://alexyu.net/plenoxels/) and [PlenOctrees](https://alexyu.net/plenoctrees/). In experiments, TensoRF with CP decomposition achieves fast reconstruction with improved rendering quality and a smaller model size compared to NeRF. Furthermore, TensoRF with VM decomposition enhances rendering quality even further, while reducing reconstruction time and maintaining a compact model size.
diff --git a/docs/quickstart/custom_dataset.md b/docs/quickstart/custom_dataset.md
index decc968cf..4792aeb72 100644
--- a/docs/quickstart/custom_dataset.md
+++ b/docs/quickstart/custom_dataset.md
@@ -106,8 +106,8 @@ cd vcpkg
```bash
git clone https://github.com/microsoft/vcpkg
cd vcpkg
-./bootstrap-vcpkg.bat
-./vcpkg install colmap[cuda]:x64-windows
+.\bootstrap-vcpkg.bat
+.\vcpkg install colmap[cuda]:x64-windows
```
:::
@@ -116,8 +116,8 @@ cd vcpkg
```bash
git clone https://github.com/microsoft/vcpkg
cd vcpkg
-./bootstrap-vcpkg.sh
-./vcpkg install colmap:x64-windows
+.\bootstrap-vcpkg.sh
+.\vcpkg install colmap:x64-windows
```
:::
@@ -355,7 +355,7 @@ Stereo equirectangular rendering for VR video is supported as omni-directional s
-To render in ODS it is essential to adjust the NeRF to have an approximately true-to-life real world scale (adjustable in the camera path) to ensure that the scene depth and IPD (distance between the eyes) is appropriate for the render to be viewable in VR. You can adjust the scene scale with the [Blender Add-on](https://docs.nerf.studio/en/latest/extensions/blender_addon.html) by appropriately scaling a point cloud representation of the NeRF.
+To render in ODS it is essential to adjust the NeRF to have an approximately true-to-life real world scale (adjustable in the camera path) to ensure that the scene depth and IPD (distance between the eyes) is appropriate for the render to be viewable in VR. You can adjust the scene scale with the [Nerfstudio Blender Add-on](https://docs.nerf.studio/en/latest/extensions/blender_addon.html) by appropriately scaling a point cloud representation of the NeRF.
Results may be unviewable if the scale is not set appropriately. The IPD is set at 64mm by default but only is accurate when the NeRF scene is true to scale.
For good quality renders, it is recommended to render at 4096x2048 per eye, or 2048x1024 per eye. Render resolutions must be specified in the camera path in a 2:1 aspect ratio. The final stacked render output will automatically be constructed as 1:1.
@@ -366,12 +366,12 @@ If you are rendering an image sequence, it is recommended to render as png inste
:::
To render with the ODS camera:
-1. Use the Nerfstudio Blender Add-on to set the scale of the NeRF scene and create the camera path
+1. Use the [Nerfstudio Blender Add-on](https://docs.nerf.studio/en/latest/extensions/blender_addon.html) to set the scale of the NeRF scene and create the camera path
- Export a point cloud representation of the NeRF
- - Import the point cloud representation in Blender and enable the Blender Add-on
+ - Import the point cloud representation in Blender and enable the Nerfstudio Blender Add-on
- Create a reference object such as a cube which may be 1x1x1 meter. You could also create a cylinder and scale it to an appropriate height of a viewer.
- Now scale the point cloud representation accordingly to match the reference object. This is to ensure that the NeRF scene is scaled as close to real life.
- - To place the camera at the correct height from the ground in the scene, you create a cylinder representing the viewer vertically scaled to the viewer’s height, and place the camera at eye level.
+ - To place the camera at the correct height from the ground in the scene, you can create a cylinder representing the viewer vertically scaled to the viewer’s height, and place the camera at eye level.
- Animate the camera movement as needed
- Create the camera path JSON file with the Nerfstudio Blender Add-on
2. Edit the JSON camera path file
@@ -392,3 +392,5 @@ If the depth of the scene is unviewable and looks too close or expanded when vie
- When rendering ODS content, Nerfstudio will first render the left eye, then the right eye, and finally vertically stack the renders. During this process, Nerfstudio will create a temporary folder to store the left and right eye renders and delete this folder once the final renders are stacked.
- If rendering content where the camera is stationary for many frames, it is recommended to only render once at that position and extend the time in a video editor since ODS renders can take a lot of time to render.
- It is recommended to render a preliminary render at a much lower resolution or frame rate to test and ensure that the depth and camera position look accurate in VR.
+ - The IPD can be modified in the `cameras.py` script as the variable `vr_ipd` (default is 64 mm).
+ - If using the Nerfstudio Blender Add-on to composite Blender objects or NeRFs with the omnidirectional render, change the Stereoscopy mode to "Parallel" set the Interocular Distance to 0.064 m.
diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py
index 47b7c419c..c9886d768 100644
--- a/nerfstudio/cameras/cameras.py
+++ b/nerfstudio/cameras/cameras.py
@@ -694,8 +694,7 @@ class Cameras(TensorDataclass):
# circle of ODS ray origins
ods_origins_circle = (
- ods_cam_position
- + isRightEye * (vr_ipd / 2.0) * (ods_x_axis.repeat(c2w.shape[1], 1)) * (torch.cos(ods_theta))[:, None]
+ isRightEye * (vr_ipd / 2.0) * (ods_x_axis.repeat(c2w.shape[1], 1)) * (torch.cos(ods_theta))[:, None]
+ isRightEye * (vr_ipd / 2.0) * (ods_z_axis.repeat(c2w.shape[1], 1)) * (torch.sin(ods_theta))[:, None]
)
diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py
index 3ff693685..3e147dcdc 100644
--- a/nerfstudio/engine/optimizers.py
+++ b/nerfstudio/engine/optimizers.py
@@ -159,3 +159,12 @@ class Optimizers:
"""
for k, v in loaded_state.items():
self.optimizers[k].load_state_dict(v)
+
+ def load_schedulers(self, loaded_state: Dict[str, Any]) -> None:
+ """Helper to load the scheduler state from previous checkpoint
+
+ Args:
+ loaded_state: the state from the previous checkpoint
+ """
+ for k, v in loaded_state.items():
+ self.schedulers[k].load_state_dict(v)
diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py
index 0ce05db0b..a89e4801c 100644
--- a/nerfstudio/engine/trainer.py
+++ b/nerfstudio/engine/trainer.py
@@ -266,7 +266,7 @@ class Trainer:
name=EventName.TRAIN_RAYS_PER_SEC,
duration=self.world_size
* self.pipeline.datamanager.get_train_rays_per_batch()
- / train_t.duration,
+ / max(0.001, train_t.duration),
step=step,
avg_over_steps=True,
)
@@ -406,6 +406,7 @@ class Trainer:
# load the checkpoints for pipeline, optimizers, and gradient scalar
self.pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"])
self.optimizers.load_optimizers(loaded_state["optimizers"])
+ self.optimizers.load_schedulers(loaded_state["schedulers"])
self.grad_scaler.load_state_dict(loaded_state["scalers"])
CONSOLE.print(f"Done loading Nerfstudio checkpoint from {load_path}")
elif load_checkpoint is not None:
@@ -439,6 +440,7 @@ class Trainer:
if hasattr(self.pipeline, "module")
else self.pipeline.state_dict(),
"optimizers": {k: v.state_dict() for (k, v) in self.optimizers.optimizers.items()},
+ "schedulers": {k: v.state_dict() for (k, v) in self.optimizers.schedulers.items()},
"scalers": self.grad_scaler.state_dict(),
},
ckpt_path,
diff --git a/nerfstudio/exporter/exporter_utils.py b/nerfstudio/exporter/exporter_utils.py
index eeaf9b6a3..3f60fc9f8 100644
--- a/nerfstudio/exporter/exporter_utils.py
+++ b/nerfstudio/exporter/exporter_utils.py
@@ -86,6 +86,7 @@ def generate_point_cloud(
num_points: int = 1000000,
remove_outliers: bool = True,
estimate_normals: bool = False,
+ reorient_normals: bool = False,
rgb_output_name: str = "rgb",
depth_output_name: str = "depth",
normal_output_name: Optional[str] = None,
@@ -100,6 +101,7 @@ def generate_point_cloud(
pipeline: Pipeline to evaluate with.
num_points: Number of points to generate. May result in less if outlier removal is used.
remove_outliers: Whether to remove outliers.
+ reorient_normals: Whether to re-orient the normals based on the view direction.
estimate_normals: Whether to estimate normals.
rgb_output_name: Name of the RGB output.
depth_output_name: Name of the depth output.
@@ -123,6 +125,8 @@ def generate_point_cloud(
points = []
rgbs = []
normals = []
+ view_directions = []
+
with progress as progress_bar:
task = progress_bar.add_task("Generating Point Cloud", total=num_points)
while not progress_bar.finished:
@@ -158,6 +162,7 @@ def generate_point_cloud(
), "Normal values from method output must be in [0, 1]"
normal = (normal * 2.0) - 1.0
point = ray_bundle.origins + ray_bundle.directions * depth
+ view_direction = ray_bundle.directions
if use_bounding_box:
comp_l = torch.tensor(bounding_box_min, device=point.device)
@@ -168,16 +173,19 @@ def generate_point_cloud(
mask = torch.all(torch.concat([point > comp_l, point < comp_m], dim=-1), dim=-1)
point = point[mask]
rgb = rgb[mask]
+ view_direction = view_direction[mask]
if normal is not None:
normal = normal[mask]
points.append(point)
rgbs.append(rgb)
+ view_directions.append(view_direction)
if normal is not None:
normals.append(normal)
progress.advance(task, point.shape[0])
points = torch.cat(points, dim=0)
rgbs = torch.cat(rgbs, dim=0)
+ view_directions = torch.cat(view_directions, dim=0).cpu()
pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(points.double().cpu().numpy())
@@ -189,6 +197,8 @@ def generate_point_cloud(
pcd, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=std_ratio)
print("\033[A\033[A")
CONSOLE.print("[bold green]:white_check_mark: Cleaning Point Cloud")
+ if ind is not None:
+ view_directions = view_directions[ind]
# either estimate_normals or normal_output_name, not both
if estimate_normals:
@@ -207,6 +217,13 @@ def generate_point_cloud(
normals = normals[ind]
pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy())
+ # re-orient the normals
+ if reorient_normals:
+ normals = torch.from_numpy(np.array(pcd.normals)).float()
+ mask = torch.sum(view_directions * normals, dim=-1) > 0
+ normals[mask] *= -1
+ pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy())
+
return pcd
diff --git a/nerfstudio/field_components/encodings.py b/nerfstudio/field_components/encodings.py
index 652d387f5..7145ae599 100644
--- a/nerfstudio/field_components/encodings.py
+++ b/nerfstudio/field_components/encodings.py
@@ -208,7 +208,12 @@ class RFFEncoding(Encoding):
self.include_input = include_input
def get_out_dim(self) -> int:
- return self.num_frequencies * 2
+ out_dim = self.num_frequencies * 2
+ if self.include_input:
+ if self.in_dim is None:
+ raise ValueError("Input dimension has not been set")
+ out_dim += self.in_dim
+ return out_dim
def forward(
self,
@@ -274,18 +279,18 @@ class HashEncoding(Encoding):
self.hash_table_size = 2**log2_hashmap_size
levels = torch.arange(num_levels)
- growth_factor = np.exp((np.log(max_res) - np.log(min_res)) / (num_levels - 1))
+ growth_factor = np.exp((np.log(max_res) - np.log(min_res)) / (num_levels - 1)) if num_levels > 1 else 1
self.scalings = torch.floor(min_res * growth_factor**levels)
self.hash_offset = levels * self.hash_table_size
- self.hash_table = torch.rand(size=(self.hash_table_size * num_levels, features_per_level)) * 2 - 1
- self.hash_table *= hash_init_scale
- self.hash_table = nn.Parameter(self.hash_table)
self.tcnn_encoding = None
+ self.hash_table = torch.empty(0)
if implementation == "tcnn" and not TCNN_EXISTS:
print_tcnn_speed_warning("HashEncoding")
- elif implementation == "tcnn":
+ implementation = "torch"
+
+ if implementation == "tcnn":
encoding_config = {
"otype": "HashGrid",
"n_levels": self.num_levels,
@@ -301,6 +306,10 @@ class HashEncoding(Encoding):
n_input_dims=3,
encoding_config=encoding_config,
)
+ elif implementation == "torch":
+ self.hash_table = torch.rand(size=(self.hash_table_size * num_levels, features_per_level)) * 2 - 1
+ self.hash_table *= hash_init_scale
+ self.hash_table = nn.Parameter(self.hash_table)
if self.tcnn_encoding is None:
assert (
diff --git a/nerfstudio/fields/sdf_field.py b/nerfstudio/fields/sdf_field.py
index a6d222933..2f730d2c5 100644
--- a/nerfstudio/fields/sdf_field.py
+++ b/nerfstudio/fields/sdf_field.py
@@ -97,19 +97,19 @@ class SDFFieldConfig(FieldConfig):
beta_init: float = 0.1
"""Init learnable beta value for transformation of sdf to density"""
encoding_type: Literal["hash", "periodic", "tensorf_vm"] = "hash"
- num_levels = 16
+ num_levels: int = 16
"""Number of encoding levels"""
- max_res = 2048
+ max_res: int = 2048
"""Maximum resolution of the encoding"""
- base_res = 16
+ base_res: int = 16
"""Base resolution of the encoding"""
- log2_hashmap_size = 19
+ log2_hashmap_size: int = 19
"""Size of the hash map"""
- features_per_level = 2
+ features_per_level: int = 2
"""Number of features per encoding level"""
- use_hash = True
+ use_hash: bool = True
"""Whether to use hash encoding"""
- smoothstep = True
+ smoothstep: bool = True
"""Whether to use the smoothstep function"""
diff --git a/nerfstudio/fields/vanilla_nerf_field.py b/nerfstudio/fields/vanilla_nerf_field.py
index 96ef0dc56..c9d2bbb78 100644
--- a/nerfstudio/fields/vanilla_nerf_field.py
+++ b/nerfstudio/fields/vanilla_nerf_field.py
@@ -15,7 +15,7 @@
"""Classic NeRF field"""
-from typing import Dict, Optional, Tuple
+from typing import Dict, Optional, Tuple, Type
import torch
from torch import Tensor, nn
@@ -57,7 +57,7 @@ class NeRFField(Field):
head_mlp_num_layers: int = 2,
head_mlp_layer_width: int = 128,
skip_connections: Tuple[int] = (4,),
- field_heads: Optional[Tuple[FieldHead]] = (RGBFieldHead(),),
+ field_heads: Optional[Tuple[Type[FieldHead]]] = (RGBFieldHead,),
use_integrated_encoding: bool = False,
spatial_distortion: Optional[SpatialDistortion] = None,
) -> None:
@@ -83,7 +83,7 @@ class NeRFField(Field):
)
self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim())
- self.field_heads = nn.ModuleList(field_heads)
+ self.field_heads = nn.ModuleList([field_head() for field_head in field_heads] if field_heads else []) # type: ignore
for field_head in self.field_heads:
field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore
diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py
index 363c8c9ea..11ba841e0 100644
--- a/nerfstudio/pipelines/base_pipeline.py
+++ b/nerfstudio/pipelines/base_pipeline.py
@@ -106,7 +106,7 @@ class Pipeline(nn.Module):
"""Returns the device that the model is on."""
return self.model.device
- def load_state_dict(self, state_dict: Mapping[str, Any], strict: bool = True):
+ def load_state_dict(self, state_dict: Mapping[str, Any], strict: Optional[bool] = None):
is_ddp_model_state = True
model_state = {}
for key, value in state_dict.items():
@@ -122,7 +122,15 @@ class Pipeline(nn.Module):
model_state = {key[len("module.") :]: value for key, value in model_state.items()}
pipeline_state = {key: value for key, value in state_dict.items() if not key.startswith("_model.")}
- self.model.load_state_dict(model_state, strict=strict)
+
+ try:
+ self.model.load_state_dict(model_state, strict=True)
+ except RuntimeError:
+ if not strict:
+ self.model.load_state_dict(model_state, strict=False)
+ else:
+ raise
+
super().load_state_dict(pipeline_state, strict=False)
@profiler.time_function
@@ -410,7 +418,7 @@ class VanillaPipeline(Pipeline):
(key[len("module.") :] if key.startswith("module.") else key): value for key, value in loaded_state.items()
}
self.model.update_to_step(step)
- self.load_state_dict(state, strict=True)
+ self.load_state_dict(state)
def get_training_callbacks(
self, training_callback_attributes: TrainingCallbackAttributes
diff --git a/nerfstudio/process_data/colmap_utils.py b/nerfstudio/process_data/colmap_utils.py
index bccdaa2f1..ac15d0937 100644
--- a/nerfstudio/process_data/colmap_utils.py
+++ b/nerfstudio/process_data/colmap_utils.py
@@ -138,7 +138,7 @@ def run_colmap(
]
if matching_method == "vocab_tree":
vocab_tree_filename = get_vocab_tree()
- feature_matcher_cmd.append(f"--VocabTreeMatching.vocab_tree_path {vocab_tree_filename}")
+ feature_matcher_cmd.append(f'--VocabTreeMatching.vocab_tree_path "{vocab_tree_filename}"')
feature_matcher_cmd = " ".join(feature_matcher_cmd)
with status(msg="[bold yellow]Running COLMAP feature matcher...", spinner="runner", verbose=verbose):
run_command(feature_matcher_cmd, verbose=verbose)
diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py
index 4e120faed..0cd413dbb 100644
--- a/nerfstudio/scripts/exporter.py
+++ b/nerfstudio/scripts/exporter.py
@@ -98,6 +98,8 @@ class ExportPointCloud(Exporter):
"""Number of points to generate. May result in less if outlier removal is used."""
remove_outliers: bool = True
"""Remove outliers from the point cloud."""
+ reorient_normals: bool = True
+ """Reorient point cloud normals based on view direction."""
normal_method: Literal["open3d", "model_output"] = "model_output"
"""Method to estimate normals with."""
normal_output_name: str = "normals"
@@ -139,6 +141,7 @@ class ExportPointCloud(Exporter):
pipeline=pipeline,
num_points=self.num_points,
remove_outliers=self.remove_outliers,
+ reorient_normals=self.reorient_normals,
estimate_normals=estimate_normals,
rgb_output_name=self.rgb_output_name,
depth_output_name=self.depth_output_name,
@@ -244,6 +247,8 @@ class ExportPoissonMesh(Exporter):
"""Number of points to generate. May result in less if outlier removal is used."""
remove_outliers: bool = True
"""Remove outliers from the point cloud."""
+ reorient_normals: bool = True
+ """Reorient point cloud normals based on view direction."""
depth_output_name: str = "depth"
"""Name of the depth output."""
rgb_output_name: str = "rgb"
@@ -297,6 +302,7 @@ class ExportPoissonMesh(Exporter):
pipeline=pipeline,
num_points=self.num_points,
remove_outliers=self.remove_outliers,
+ reorient_normals=self.reorient_normals,
estimate_normals=estimate_normals,
rgb_output_name=self.rgb_output_name,
depth_output_name=self.depth_output_name,
diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py
index aef5d4909..60378616d 100644
--- a/nerfstudio/scripts/render.py
+++ b/nerfstudio/scripts/render.py
@@ -469,6 +469,7 @@ class RenderInterpolated(BaseRender):
rendered_resolution_scaling_factor=1.0 / self.downscale_factor,
seconds=seconds,
output_format=self.output_format,
+ image_format=self.image_format,
colormap_options=self.colormap_options,
)
@@ -511,6 +512,7 @@ class SpiralRender(BaseRender):
rendered_resolution_scaling_factor=1.0 / self.downscale_factor,
seconds=self.seconds,
output_format=self.output_format,
+ image_format=self.image_format,
colormap_options=self.colormap_options,
)
diff --git a/nerfstudio/scripts/viewer/run_viewer.py b/nerfstudio/scripts/viewer/run_viewer.py
index 0600a9107..c4ebefacb 100644
--- a/nerfstudio/scripts/viewer/run_viewer.py
+++ b/nerfstudio/scripts/viewer/run_viewer.py
@@ -22,6 +22,8 @@ import time
from dataclasses import dataclass, field, fields
from pathlib import Path
+from typing import Literal
+
import tyro
from nerfstudio.configs.base_config import ViewerConfig
@@ -30,6 +32,7 @@ from nerfstudio.pipelines.base_pipeline import Pipeline
from nerfstudio.utils import writer
from nerfstudio.utils.eval_utils import eval_setup
from nerfstudio.viewer.server.viewer_state import ViewerState
+from nerfstudio.viewer_beta.viewer import Viewer as ViewerBetaState
@dataclass
@@ -51,6 +54,8 @@ class RunViewer:
"""Path to config YAML file."""
viewer: ViewerConfigWithoutNumRays = field(default_factory=ViewerConfigWithoutNumRays)
"""Viewer configuration"""
+ vis: Literal["viewer", "viewer_beta"] = "viewer"
+ """Type of viewer"""
def main(self) -> None:
"""Main function."""
@@ -61,7 +66,7 @@ class RunViewer:
)
num_rays_per_chunk = config.viewer.num_rays_per_chunk
assert self.viewer.num_rays_per_chunk == -1
- config.vis = "viewer"
+ config.vis = self.vis
config.viewer = self.viewer.as_viewer_config()
config.viewer.num_rays_per_chunk = num_rays_per_chunk
@@ -83,13 +88,24 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int):
"""
base_dir = config.get_base_dir()
viewer_log_path = base_dir / config.viewer.relative_log_filename
- viewer_state = ViewerState(
- config.viewer,
- log_filename=viewer_log_path,
- datapath=pipeline.datamanager.get_datapath(),
- pipeline=pipeline,
- )
- banner_messages = [f"Viewer at: {viewer_state.viewer_url}"]
+ banner_messages = None
+ viewer_state = None
+ if config.vis == "viewer":
+ viewer_state = ViewerState(
+ config.viewer,
+ log_filename=viewer_log_path,
+ datapath=pipeline.datamanager.get_datapath(),
+ pipeline=pipeline,
+ )
+ banner_messages = [f"Viewer at: {viewer_state.viewer_url}"]
+ if config.vis == "viewer_beta":
+ viewer_state = ViewerBetaState(
+ config.viewer,
+ log_filename=viewer_log_path,
+ datapath=base_dir,
+ pipeline=pipeline,
+ )
+ banner_messages = [f"Viewer Beta at: {viewer_state.viewer_url}"]
# We don't need logging, but writer.GLOBAL_BUFFER needs to be populated
config.logging.local_writer.enable = False
@@ -100,7 +116,8 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int):
dataset=pipeline.datamanager.train_dataset,
train_state="completed",
)
- viewer_state.viser_server.set_training_state("completed")
+ if isinstance(viewer_state, ViewerState):
+ viewer_state.viser_server.set_training_state("completed")
viewer_state.update_scene(step=step)
while True:
time.sleep(0.01)
diff --git a/nerfstudio/viewer/server/render_state_machine.py b/nerfstudio/viewer/server/render_state_machine.py
index 540a8378b..390390d48 100644
--- a/nerfstudio/viewer/server/render_state_machine.py
+++ b/nerfstudio/viewer/server/render_state_machine.py
@@ -243,6 +243,8 @@ class RenderStateMachine(threading.Thread):
image_height: the maximum image height that can be rendered in the time budget
image_width: the maximum image width that can be rendered in the time budget
"""
+ if aspect_ratio == 0:
+ aspect_ratio = 0.001
max_res = self.viewer.control_panel.max_res
if self.state == "high":
# high res is always static
diff --git a/nerfstudio/viewer_beta/viewer.py b/nerfstudio/viewer_beta/viewer.py
index 51ca6ab14..4c007f082 100644
--- a/nerfstudio/viewer_beta/viewer.py
+++ b/nerfstudio/viewer_beta/viewer.py
@@ -118,8 +118,8 @@ class Viewer:
def _(_: viser.CameraHandle) -> None:
R = vtf.SO3(wxyz=client.camera.wxyz)
R = R @ vtf.SO3.from_x_radians(np.pi)
- R = torch.tensor(R.as_matrix())
- pos = torch.tensor(client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO
+ R = torch.tensor(R.as_matrix(), dtype=torch.float32)
+ pos = torch.tensor(client.camera.position, dtype=torch.float32) / VISER_NERFSTUDIO_SCALE_RATIO
c2w = torch.concatenate([R, pos[:, None]], dim=1)
self.camera_state = CameraState(fov=client.camera.fov, aspect=client.camera.aspect, c2w=c2w)
self.render_statemachine.action(RenderAction("move", self.camera_state))