mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 06:14:03 +08:00
Merge branch 'main' into fix_gener_mesh_export
This commit is contained in:
20 files changed
+140
-50
No files matched your search
+4
-5
@@ -125,9 +125,8 @@ RUN CUDA_VER=${CUDA_VERSION%.*} && CUDA_VER=${CUDA_VER//./} && python3.10 -m pip
|
||||
ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}
|
||||
RUN python3.10 -m pip install git+https://github.com/NVlabs/tiny-cuda-nn.git@v1.6#subdirectory=bindings/torch
|
||||
|
||||
# Install pycolmap 0.3.0, required by hloc.
|
||||
# TODO(https://github.com/colmap/pycolmap/issues/111) use wheel when available for Python 3.10
|
||||
RUN git clone --branch v0.3.0 --recursive https://github.com/colmap/pycolmap.git && \
|
||||
# Install pycolmap, required by hloc.
|
||||
RUN git clone --branch v0.4.0 --recursive https://github.com/colmap/pycolmap.git && \
|
||||
cd pycolmap && \
|
||||
python3.10 -m pip install . && \
|
||||
cd ..
|
||||
@@ -139,13 +138,13 @@ RUN git clone --branch master --recursive https://github.com/cvg/Hierarchical-Lo
|
||||
cd ..
|
||||
|
||||
# Install pyceres from source
|
||||
RUN git clone --branch main --recursive https://github.com/cvg/pyceres.git && \
|
||||
RUN git clone --branch v1.0 --recursive https://github.com/cvg/pyceres.git && \
|
||||
cd pyceres && \
|
||||
python3.10 -m pip install -e . && \
|
||||
cd ..
|
||||
|
||||
# Install pixel perfect sfm.
|
||||
RUN git clone --branch main --recursive https://github.com/cvg/pixel-perfect-sfm.git && \
|
||||
RUN git clone --branch v1.0 --recursive https://github.com/cvg/pixel-perfect-sfm.git && \
|
||||
cd pixel-perfect-sfm && \
|
||||
python3.10 -m pip install -e . && \
|
||||
cd ..
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
|
||||
## Overview
|
||||
|
||||
This Blender add-on allows for compositing with a Nerfstudio render as a background layer by generating a camera path JSON file from the Blender camera path, as well as a way to import Nerfstudio JSON files as a Blender camera baked with the Nerfstudio camera path. This add-on also allows compositing multiple NeRF objects into a NeRF scene. This is achieved by importing a mesh or point-cloud representation of the NeRF scene from Nerfstudio into Blender and getting the camera coordinates relative to the transformations of the NeRF representation. Dynamic FOV from the Blender camera is supported and will match the Nerfstudio render. Perspective and equirectangular cameras are supported.
|
||||
This Blender add-on allows for compositing with a Nerfstudio render as a background layer by generating a camera path JSON file from the Blender camera path, as well as a way to import Nerfstudio JSON files as a Blender camera baked with the Nerfstudio camera path. This add-on also allows compositing multiple NeRF objects into a NeRF scene. This is achieved by importing a mesh or point-cloud representation of the NeRF scene from Nerfstudio into Blender and getting the camera coordinates relative to the transformations of the NeRF representation. Dynamic FOV from the Blender camera is supported and will match the Nerfstudio render. Perspective, equirectangular, and omnidirectional stereo (VR 360) cameras are supported.
|
||||
|
||||
<center>
|
||||
<img width="800" alt="image" src="https://user-images.githubusercontent.com/9502341/211442247-99d1ebc7-3ef9-46f7-9bcc-0e18553f19b7.PNG">
|
||||
@@ -89,7 +89,7 @@ This Blender add-on allows for compositing with a Nerfstudio render as a backgro
|
||||
- The generated JSON camera path follows the user specified frame start, end, and step fields in the Output Properties in Blender. The JSON file specifies the user specified x and y render resolutions at the given % in the Output Properties.
|
||||
- The add-on computes the camera coordinates based on the active camera in the scene.
|
||||
- FOV animated changes of the camera will be matched with the NeRF render.
|
||||
- Perspective and equirectangular cameras are supported and can be selected within Blender.
|
||||
- Perspective, equirectangular, and omnidirectional stereo cameras are supported and can be selected within Blender.
|
||||
- The generated JSON file can be imported into Nerfstudio. Each keyframe of the camera transform in the frame sequence in Blender will be a keyframe in Nerfstudio. The exported JSON camera path is baked, where each frame in the sequence is a keyframe. This is to ensure that frame interpolation across Nerfstudio and Blender do not differ.
|
||||
<center>
|
||||
<img width="800" alt="image" src="https://user-images.githubusercontent.com/9502341/211245108-587a97f7-d48d-4515-b09a-5644fd966298.png">
|
||||
@@ -101,6 +101,10 @@ This Blender add-on allows for compositing with a Nerfstudio render as a backgro
|
||||
- It is recommended to export a high fidelity mesh as the NeRF representation from Nerfstudio. However if that is not possible or the scene is too large, a point cloud representation also works.
|
||||
- For compositing, it is recommended to convert the video render into image frames to ensure the Blender and NeRF renders are in sync.
|
||||
- Currently, dynamic camera focus is not supported.
|
||||
- For compositing Blender objects or NeRFs with Nerfstudio omnidirectional stereo renders (VR 360), configure the Blender camera to be equirectangular and enable stereoscopy in the Output Properties. Under the Stereoscopy panel the Blender camera settings, change the mode to "Parallel", set the Interocular Distance to 0.064 m, and checkmark "Spherical Stereo".
|
||||
<center>
|
||||
<img width="300" alt="image" src="https://github-production-user-asset-6210df.s3.amazonaws.com/9502341/253217833-fd607601-2b81-48ab-ac5d-e55514a588da.png">
|
||||
</center>
|
||||
- Fisheye and orthographic cameras are not supported.
|
||||
- A walkthrough of this section is included in the tutorial video.
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 43 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 273 KiB |
@@ -8,6 +8,10 @@
|
||||
Paper Website
|
||||
```
|
||||
|
||||
<video id="teaser" muted autoplay playsinline loop controls width="100%">
|
||||
<source id="mp4" src="https://apchenstu.github.io/TensoRF/video/train_process.mp4" type="video/mp4">
|
||||
</video>
|
||||
|
||||
### Running Model
|
||||
|
||||
```bash
|
||||
@@ -16,4 +20,14 @@ ns-train tensorf
|
||||
|
||||
## Overview
|
||||
|
||||
🚧 Guide under construction 🚧
|
||||
```{image} imgs/tensorf/models_tensorf_pipeline.png
|
||||
:align: center
|
||||
```
|
||||
|
||||
TensoRF models the radiance field of a scene as a 4D tensor, which represents a 3D voxel grid with per-voxel multi-channel features. TensoRF factorizes the 4D scene tensor into multiple compact low-rank tensor components using CP or VM modes. CP decomposition factorizes tensors into rank-one components with compact vectors. Vector-Matrix (VM) decomposition factorizes tensors into compact vector and matrix factors.
|
||||
|
||||
```{image} imgs/tensorf/models_tensorf_factorization.png
|
||||
:align: center
|
||||
```
|
||||
|
||||
TensoRF with CP(left) and VM(right) decompositions results in a significantly reduced memory footprint compared to previous and concurrent works that directly optimize per-voxel features, such as [Plenoxels](https://alexyu.net/plenoxels/) and [PlenOctrees](https://alexyu.net/plenoctrees/). In experiments, TensoRF with CP decomposition achieves fast reconstruction with improved rendering quality and a smaller model size compared to NeRF. Furthermore, TensoRF with VM decomposition enhances rendering quality even further, while reducing reconstruction time and maintaining a compact model size.
|
||||
@@ -106,8 +106,8 @@ cd vcpkg
|
||||
```bash
|
||||
git clone https://github.com/microsoft/vcpkg
|
||||
cd vcpkg
|
||||
./bootstrap-vcpkg.bat
|
||||
./vcpkg install colmap[cuda]:x64-windows
|
||||
.\bootstrap-vcpkg.bat
|
||||
.\vcpkg install colmap[cuda]:x64-windows
|
||||
```
|
||||
|
||||
:::
|
||||
@@ -116,8 +116,8 @@ cd vcpkg
|
||||
```bash
|
||||
git clone https://github.com/microsoft/vcpkg
|
||||
cd vcpkg
|
||||
./bootstrap-vcpkg.sh
|
||||
./vcpkg install colmap:x64-windows
|
||||
.\bootstrap-vcpkg.sh
|
||||
.\vcpkg install colmap:x64-windows
|
||||
```
|
||||
|
||||
:::
|
||||
@@ -355,7 +355,7 @@ Stereo equirectangular rendering for VR video is supported as omni-directional s
|
||||
<img img width="300" src="https://github-production-user-asset-6210df.s3.amazonaws.com/9502341/240530353-5c05d12b-6e42-42bb-8ff5-f54aa30ca262.jpg">
|
||||
</center>
|
||||
|
||||
To render in ODS it is essential to adjust the NeRF to have an approximately true-to-life real world scale (adjustable in the camera path) to ensure that the scene depth and IPD (distance between the eyes) is appropriate for the render to be viewable in VR. You can adjust the scene scale with the [Blender Add-on](https://docs.nerf.studio/en/latest/extensions/blender_addon.html) by appropriately scaling a point cloud representation of the NeRF.
|
||||
To render in ODS it is essential to adjust the NeRF to have an approximately true-to-life real world scale (adjustable in the camera path) to ensure that the scene depth and IPD (distance between the eyes) is appropriate for the render to be viewable in VR. You can adjust the scene scale with the [Nerfstudio Blender Add-on](https://docs.nerf.studio/en/latest/extensions/blender_addon.html) by appropriately scaling a point cloud representation of the NeRF.
|
||||
Results may be unviewable if the scale is not set appropriately. The IPD is set at 64mm by default but only is accurate when the NeRF scene is true to scale.
|
||||
|
||||
For good quality renders, it is recommended to render at 4096x2048 per eye, or 2048x1024 per eye. Render resolutions must be specified in the camera path in a 2:1 aspect ratio. The final stacked render output will automatically be constructed as 1:1.
|
||||
@@ -366,12 +366,12 @@ If you are rendering an image sequence, it is recommended to render as png inste
|
||||
:::
|
||||
|
||||
To render with the ODS camera:
|
||||
1. Use the Nerfstudio Blender Add-on to set the scale of the NeRF scene and create the camera path
|
||||
1. Use the [Nerfstudio Blender Add-on](https://docs.nerf.studio/en/latest/extensions/blender_addon.html) to set the scale of the NeRF scene and create the camera path
|
||||
- Export a point cloud representation of the NeRF
|
||||
- Import the point cloud representation in Blender and enable the Blender Add-on
|
||||
- Import the point cloud representation in Blender and enable the Nerfstudio Blender Add-on
|
||||
- Create a reference object such as a cube which may be 1x1x1 meter. You could also create a cylinder and scale it to an appropriate height of a viewer.
|
||||
- Now scale the point cloud representation accordingly to match the reference object. This is to ensure that the NeRF scene is scaled as close to real life.
|
||||
- To place the camera at the correct height from the ground in the scene, you create a cylinder representing the viewer vertically scaled to the viewer’s height, and place the camera at eye level.
|
||||
- To place the camera at the correct height from the ground in the scene, you can create a cylinder representing the viewer vertically scaled to the viewer’s height, and place the camera at eye level.
|
||||
- Animate the camera movement as needed
|
||||
- Create the camera path JSON file with the Nerfstudio Blender Add-on
|
||||
2. Edit the JSON camera path file
|
||||
@@ -392,3 +392,5 @@ If the depth of the scene is unviewable and looks too close or expanded when vie
|
||||
- When rendering ODS content, Nerfstudio will first render the left eye, then the right eye, and finally vertically stack the renders. During this process, Nerfstudio will create a temporary folder to store the left and right eye renders and delete this folder once the final renders are stacked.
|
||||
- If rendering content where the camera is stationary for many frames, it is recommended to only render once at that position and extend the time in a video editor since ODS renders can take a lot of time to render.
|
||||
- It is recommended to render a preliminary render at a much lower resolution or frame rate to test and ensure that the depth and camera position look accurate in VR.
|
||||
- The IPD can be modified in the `cameras.py` script as the variable `vr_ipd` (default is 64 mm).
|
||||
- If using the Nerfstudio Blender Add-on to composite Blender objects or NeRFs with the omnidirectional render, change the Stereoscopy mode to "Parallel" set the Interocular Distance to 0.064 m.
|
||||
@@ -694,8 +694,7 @@ class Cameras(TensorDataclass):
|
||||
|
||||
# circle of ODS ray origins
|
||||
ods_origins_circle = (
|
||||
ods_cam_position
|
||||
+ isRightEye * (vr_ipd / 2.0) * (ods_x_axis.repeat(c2w.shape[1], 1)) * (torch.cos(ods_theta))[:, None]
|
||||
isRightEye * (vr_ipd / 2.0) * (ods_x_axis.repeat(c2w.shape[1], 1)) * (torch.cos(ods_theta))[:, None]
|
||||
+ isRightEye * (vr_ipd / 2.0) * (ods_z_axis.repeat(c2w.shape[1], 1)) * (torch.sin(ods_theta))[:, None]
|
||||
)
|
||||
|
||||
|
||||
@@ -159,3 +159,12 @@ class Optimizers:
|
||||
"""
|
||||
for k, v in loaded_state.items():
|
||||
self.optimizers[k].load_state_dict(v)
|
||||
|
||||
def load_schedulers(self, loaded_state: Dict[str, Any]) -> None:
|
||||
"""Helper to load the scheduler state from previous checkpoint
|
||||
|
||||
Args:
|
||||
loaded_state: the state from the previous checkpoint
|
||||
"""
|
||||
for k, v in loaded_state.items():
|
||||
self.schedulers[k].load_state_dict(v)
|
||||
@@ -266,7 +266,7 @@ class Trainer:
|
||||
name=EventName.TRAIN_RAYS_PER_SEC,
|
||||
duration=self.world_size
|
||||
* self.pipeline.datamanager.get_train_rays_per_batch()
|
||||
/ train_t.duration,
|
||||
/ max(0.001, train_t.duration),
|
||||
step=step,
|
||||
avg_over_steps=True,
|
||||
)
|
||||
@@ -406,6 +406,7 @@ class Trainer:
|
||||
# load the checkpoints for pipeline, optimizers, and gradient scalar
|
||||
self.pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"])
|
||||
self.optimizers.load_optimizers(loaded_state["optimizers"])
|
||||
self.optimizers.load_schedulers(loaded_state["schedulers"])
|
||||
self.grad_scaler.load_state_dict(loaded_state["scalers"])
|
||||
CONSOLE.print(f"Done loading Nerfstudio checkpoint from {load_path}")
|
||||
elif load_checkpoint is not None:
|
||||
@@ -439,6 +440,7 @@ class Trainer:
|
||||
if hasattr(self.pipeline, "module")
|
||||
else self.pipeline.state_dict(),
|
||||
"optimizers": {k: v.state_dict() for (k, v) in self.optimizers.optimizers.items()},
|
||||
"schedulers": {k: v.state_dict() for (k, v) in self.optimizers.schedulers.items()},
|
||||
"scalers": self.grad_scaler.state_dict(),
|
||||
},
|
||||
ckpt_path,
|
||||
|
||||
@@ -86,6 +86,7 @@ def generate_point_cloud(
|
||||
num_points: int = 1000000,
|
||||
remove_outliers: bool = True,
|
||||
estimate_normals: bool = False,
|
||||
reorient_normals: bool = False,
|
||||
rgb_output_name: str = "rgb",
|
||||
depth_output_name: str = "depth",
|
||||
normal_output_name: Optional[str] = None,
|
||||
@@ -100,6 +101,7 @@ def generate_point_cloud(
|
||||
pipeline: Pipeline to evaluate with.
|
||||
num_points: Number of points to generate. May result in less if outlier removal is used.
|
||||
remove_outliers: Whether to remove outliers.
|
||||
reorient_normals: Whether to re-orient the normals based on the view direction.
|
||||
estimate_normals: Whether to estimate normals.
|
||||
rgb_output_name: Name of the RGB output.
|
||||
depth_output_name: Name of the depth output.
|
||||
@@ -123,6 +125,8 @@ def generate_point_cloud(
|
||||
points = []
|
||||
rgbs = []
|
||||
normals = []
|
||||
view_directions = []
|
||||
|
||||
with progress as progress_bar:
|
||||
task = progress_bar.add_task("Generating Point Cloud", total=num_points)
|
||||
while not progress_bar.finished:
|
||||
@@ -158,6 +162,7 @@ def generate_point_cloud(
|
||||
), "Normal values from method output must be in [0, 1]"
|
||||
normal = (normal * 2.0) - 1.0
|
||||
point = ray_bundle.origins + ray_bundle.directions * depth
|
||||
view_direction = ray_bundle.directions
|
||||
|
||||
if use_bounding_box:
|
||||
comp_l = torch.tensor(bounding_box_min, device=point.device)
|
||||
@@ -168,16 +173,19 @@ def generate_point_cloud(
|
||||
mask = torch.all(torch.concat([point > comp_l, point < comp_m], dim=-1), dim=-1)
|
||||
point = point[mask]
|
||||
rgb = rgb[mask]
|
||||
view_direction = view_direction[mask]
|
||||
if normal is not None:
|
||||
normal = normal[mask]
|
||||
|
||||
points.append(point)
|
||||
rgbs.append(rgb)
|
||||
view_directions.append(view_direction)
|
||||
if normal is not None:
|
||||
normals.append(normal)
|
||||
progress.advance(task, point.shape[0])
|
||||
points = torch.cat(points, dim=0)
|
||||
rgbs = torch.cat(rgbs, dim=0)
|
||||
view_directions = torch.cat(view_directions, dim=0).cpu()
|
||||
|
||||
pcd = o3d.geometry.PointCloud()
|
||||
pcd.points = o3d.utility.Vector3dVector(points.double().cpu().numpy())
|
||||
@@ -189,6 +197,8 @@ def generate_point_cloud(
|
||||
pcd, ind = pcd.remove_statistical_outlier(nb_neighbors=20, std_ratio=std_ratio)
|
||||
print("\033[A\033[A")
|
||||
CONSOLE.print("[bold green]:white_check_mark: Cleaning Point Cloud")
|
||||
if ind is not None:
|
||||
view_directions = view_directions[ind]
|
||||
|
||||
# either estimate_normals or normal_output_name, not both
|
||||
if estimate_normals:
|
||||
@@ -207,6 +217,13 @@ def generate_point_cloud(
|
||||
normals = normals[ind]
|
||||
pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy())
|
||||
|
||||
# re-orient the normals
|
||||
if reorient_normals:
|
||||
normals = torch.from_numpy(np.array(pcd.normals)).float()
|
||||
mask = torch.sum(view_directions * normals, dim=-1) > 0
|
||||
normals[mask] *= -1
|
||||
pcd.normals = o3d.utility.Vector3dVector(normals.double().cpu().numpy())
|
||||
|
||||
return pcd
|
||||
|
||||
|
||||
|
||||
@@ -208,7 +208,12 @@ class RFFEncoding(Encoding):
|
||||
self.include_input = include_input
|
||||
|
||||
def get_out_dim(self) -> int:
|
||||
return self.num_frequencies * 2
|
||||
out_dim = self.num_frequencies * 2
|
||||
if self.include_input:
|
||||
if self.in_dim is None:
|
||||
raise ValueError("Input dimension has not been set")
|
||||
out_dim += self.in_dim
|
||||
return out_dim
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -274,18 +279,18 @@ class HashEncoding(Encoding):
|
||||
self.hash_table_size = 2**log2_hashmap_size
|
||||
|
||||
levels = torch.arange(num_levels)
|
||||
growth_factor = np.exp((np.log(max_res) - np.log(min_res)) / (num_levels - 1))
|
||||
growth_factor = np.exp((np.log(max_res) - np.log(min_res)) / (num_levels - 1)) if num_levels > 1 else 1
|
||||
self.scalings = torch.floor(min_res * growth_factor**levels)
|
||||
|
||||
self.hash_offset = levels * self.hash_table_size
|
||||
self.hash_table = torch.rand(size=(self.hash_table_size * num_levels, features_per_level)) * 2 - 1
|
||||
self.hash_table *= hash_init_scale
|
||||
self.hash_table = nn.Parameter(self.hash_table)
|
||||
|
||||
self.tcnn_encoding = None
|
||||
self.hash_table = torch.empty(0)
|
||||
if implementation == "tcnn" and not TCNN_EXISTS:
|
||||
print_tcnn_speed_warning("HashEncoding")
|
||||
elif implementation == "tcnn":
|
||||
implementation = "torch"
|
||||
|
||||
if implementation == "tcnn":
|
||||
encoding_config = {
|
||||
"otype": "HashGrid",
|
||||
"n_levels": self.num_levels,
|
||||
@@ -301,6 +306,10 @@ class HashEncoding(Encoding):
|
||||
n_input_dims=3,
|
||||
encoding_config=encoding_config,
|
||||
)
|
||||
elif implementation == "torch":
|
||||
self.hash_table = torch.rand(size=(self.hash_table_size * num_levels, features_per_level)) * 2 - 1
|
||||
self.hash_table *= hash_init_scale
|
||||
self.hash_table = nn.Parameter(self.hash_table)
|
||||
|
||||
if self.tcnn_encoding is None:
|
||||
assert (
|
||||
|
||||
@@ -97,19 +97,19 @@ class SDFFieldConfig(FieldConfig):
|
||||
beta_init: float = 0.1
|
||||
"""Init learnable beta value for transformation of sdf to density"""
|
||||
encoding_type: Literal["hash", "periodic", "tensorf_vm"] = "hash"
|
||||
num_levels = 16
|
||||
num_levels: int = 16
|
||||
"""Number of encoding levels"""
|
||||
max_res = 2048
|
||||
max_res: int = 2048
|
||||
"""Maximum resolution of the encoding"""
|
||||
base_res = 16
|
||||
base_res: int = 16
|
||||
"""Base resolution of the encoding"""
|
||||
log2_hashmap_size = 19
|
||||
log2_hashmap_size: int = 19
|
||||
"""Size of the hash map"""
|
||||
features_per_level = 2
|
||||
features_per_level: int = 2
|
||||
"""Number of features per encoding level"""
|
||||
use_hash = True
|
||||
use_hash: bool = True
|
||||
"""Whether to use hash encoding"""
|
||||
smoothstep = True
|
||||
smoothstep: bool = True
|
||||
"""Whether to use the smoothstep function"""
|
||||
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
"""Classic NeRF field"""
|
||||
|
||||
|
||||
from typing import Dict, Optional, Tuple
|
||||
from typing import Dict, Optional, Tuple, Type
|
||||
|
||||
import torch
|
||||
from torch import Tensor, nn
|
||||
@@ -57,7 +57,7 @@ class NeRFField(Field):
|
||||
head_mlp_num_layers: int = 2,
|
||||
head_mlp_layer_width: int = 128,
|
||||
skip_connections: Tuple[int] = (4,),
|
||||
field_heads: Optional[Tuple[FieldHead]] = (RGBFieldHead(),),
|
||||
field_heads: Optional[Tuple[Type[FieldHead]]] = (RGBFieldHead,),
|
||||
use_integrated_encoding: bool = False,
|
||||
spatial_distortion: Optional[SpatialDistortion] = None,
|
||||
) -> None:
|
||||
@@ -83,7 +83,7 @@ class NeRFField(Field):
|
||||
)
|
||||
|
||||
self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim())
|
||||
self.field_heads = nn.ModuleList(field_heads)
|
||||
self.field_heads = nn.ModuleList([field_head() for field_head in field_heads] if field_heads else []) # type: ignore
|
||||
for field_head in self.field_heads:
|
||||
field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ class Pipeline(nn.Module):
|
||||
"""Returns the device that the model is on."""
|
||||
return self.model.device
|
||||
|
||||
def load_state_dict(self, state_dict: Mapping[str, Any], strict: bool = True):
|
||||
def load_state_dict(self, state_dict: Mapping[str, Any], strict: Optional[bool] = None):
|
||||
is_ddp_model_state = True
|
||||
model_state = {}
|
||||
for key, value in state_dict.items():
|
||||
@@ -122,7 +122,15 @@ class Pipeline(nn.Module):
|
||||
model_state = {key[len("module.") :]: value for key, value in model_state.items()}
|
||||
|
||||
pipeline_state = {key: value for key, value in state_dict.items() if not key.startswith("_model.")}
|
||||
self.model.load_state_dict(model_state, strict=strict)
|
||||
|
||||
try:
|
||||
self.model.load_state_dict(model_state, strict=True)
|
||||
except RuntimeError:
|
||||
if not strict:
|
||||
self.model.load_state_dict(model_state, strict=False)
|
||||
else:
|
||||
raise
|
||||
|
||||
super().load_state_dict(pipeline_state, strict=False)
|
||||
|
||||
@profiler.time_function
|
||||
@@ -410,7 +418,7 @@ class VanillaPipeline(Pipeline):
|
||||
(key[len("module.") :] if key.startswith("module.") else key): value for key, value in loaded_state.items()
|
||||
}
|
||||
self.model.update_to_step(step)
|
||||
self.load_state_dict(state, strict=True)
|
||||
self.load_state_dict(state)
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes
|
||||
|
||||
@@ -138,7 +138,7 @@ def run_colmap(
|
||||
]
|
||||
if matching_method == "vocab_tree":
|
||||
vocab_tree_filename = get_vocab_tree()
|
||||
feature_matcher_cmd.append(f"--VocabTreeMatching.vocab_tree_path {vocab_tree_filename}")
|
||||
feature_matcher_cmd.append(f'--VocabTreeMatching.vocab_tree_path "{vocab_tree_filename}"')
|
||||
feature_matcher_cmd = " ".join(feature_matcher_cmd)
|
||||
with status(msg="[bold yellow]Running COLMAP feature matcher...", spinner="runner", verbose=verbose):
|
||||
run_command(feature_matcher_cmd, verbose=verbose)
|
||||
|
||||
@@ -98,6 +98,8 @@ class ExportPointCloud(Exporter):
|
||||
"""Number of points to generate. May result in less if outlier removal is used."""
|
||||
remove_outliers: bool = True
|
||||
"""Remove outliers from the point cloud."""
|
||||
reorient_normals: bool = True
|
||||
"""Reorient point cloud normals based on view direction."""
|
||||
normal_method: Literal["open3d", "model_output"] = "model_output"
|
||||
"""Method to estimate normals with."""
|
||||
normal_output_name: str = "normals"
|
||||
@@ -139,6 +141,7 @@ class ExportPointCloud(Exporter):
|
||||
pipeline=pipeline,
|
||||
num_points=self.num_points,
|
||||
remove_outliers=self.remove_outliers,
|
||||
reorient_normals=self.reorient_normals,
|
||||
estimate_normals=estimate_normals,
|
||||
rgb_output_name=self.rgb_output_name,
|
||||
depth_output_name=self.depth_output_name,
|
||||
@@ -244,6 +247,8 @@ class ExportPoissonMesh(Exporter):
|
||||
"""Number of points to generate. May result in less if outlier removal is used."""
|
||||
remove_outliers: bool = True
|
||||
"""Remove outliers from the point cloud."""
|
||||
reorient_normals: bool = True
|
||||
"""Reorient point cloud normals based on view direction."""
|
||||
depth_output_name: str = "depth"
|
||||
"""Name of the depth output."""
|
||||
rgb_output_name: str = "rgb"
|
||||
@@ -297,6 +302,7 @@ class ExportPoissonMesh(Exporter):
|
||||
pipeline=pipeline,
|
||||
num_points=self.num_points,
|
||||
remove_outliers=self.remove_outliers,
|
||||
reorient_normals=self.reorient_normals,
|
||||
estimate_normals=estimate_normals,
|
||||
rgb_output_name=self.rgb_output_name,
|
||||
depth_output_name=self.depth_output_name,
|
||||
|
||||
@@ -469,6 +469,7 @@ class RenderInterpolated(BaseRender):
|
||||
rendered_resolution_scaling_factor=1.0 / self.downscale_factor,
|
||||
seconds=seconds,
|
||||
output_format=self.output_format,
|
||||
image_format=self.image_format,
|
||||
colormap_options=self.colormap_options,
|
||||
)
|
||||
|
||||
@@ -511,6 +512,7 @@ class SpiralRender(BaseRender):
|
||||
rendered_resolution_scaling_factor=1.0 / self.downscale_factor,
|
||||
seconds=self.seconds,
|
||||
output_format=self.output_format,
|
||||
image_format=self.image_format,
|
||||
colormap_options=self.colormap_options,
|
||||
)
|
||||
|
||||
|
||||
@@ -22,6 +22,8 @@ import time
|
||||
from dataclasses import dataclass, field, fields
|
||||
from pathlib import Path
|
||||
|
||||
from typing import Literal
|
||||
|
||||
import tyro
|
||||
|
||||
from nerfstudio.configs.base_config import ViewerConfig
|
||||
@@ -30,6 +32,7 @@ from nerfstudio.pipelines.base_pipeline import Pipeline
|
||||
from nerfstudio.utils import writer
|
||||
from nerfstudio.utils.eval_utils import eval_setup
|
||||
from nerfstudio.viewer.server.viewer_state import ViewerState
|
||||
from nerfstudio.viewer_beta.viewer import Viewer as ViewerBetaState
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -51,6 +54,8 @@ class RunViewer:
|
||||
"""Path to config YAML file."""
|
||||
viewer: ViewerConfigWithoutNumRays = field(default_factory=ViewerConfigWithoutNumRays)
|
||||
"""Viewer configuration"""
|
||||
vis: Literal["viewer", "viewer_beta"] = "viewer"
|
||||
"""Type of viewer"""
|
||||
|
||||
def main(self) -> None:
|
||||
"""Main function."""
|
||||
@@ -61,7 +66,7 @@ class RunViewer:
|
||||
)
|
||||
num_rays_per_chunk = config.viewer.num_rays_per_chunk
|
||||
assert self.viewer.num_rays_per_chunk == -1
|
||||
config.vis = "viewer"
|
||||
config.vis = self.vis
|
||||
config.viewer = self.viewer.as_viewer_config()
|
||||
config.viewer.num_rays_per_chunk = num_rays_per_chunk
|
||||
|
||||
@@ -83,13 +88,24 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int):
|
||||
"""
|
||||
base_dir = config.get_base_dir()
|
||||
viewer_log_path = base_dir / config.viewer.relative_log_filename
|
||||
viewer_state = ViewerState(
|
||||
config.viewer,
|
||||
log_filename=viewer_log_path,
|
||||
datapath=pipeline.datamanager.get_datapath(),
|
||||
pipeline=pipeline,
|
||||
)
|
||||
banner_messages = [f"Viewer at: {viewer_state.viewer_url}"]
|
||||
banner_messages = None
|
||||
viewer_state = None
|
||||
if config.vis == "viewer":
|
||||
viewer_state = ViewerState(
|
||||
config.viewer,
|
||||
log_filename=viewer_log_path,
|
||||
datapath=pipeline.datamanager.get_datapath(),
|
||||
pipeline=pipeline,
|
||||
)
|
||||
banner_messages = [f"Viewer at: {viewer_state.viewer_url}"]
|
||||
if config.vis == "viewer_beta":
|
||||
viewer_state = ViewerBetaState(
|
||||
config.viewer,
|
||||
log_filename=viewer_log_path,
|
||||
datapath=base_dir,
|
||||
pipeline=pipeline,
|
||||
)
|
||||
banner_messages = [f"Viewer Beta at: {viewer_state.viewer_url}"]
|
||||
|
||||
# We don't need logging, but writer.GLOBAL_BUFFER needs to be populated
|
||||
config.logging.local_writer.enable = False
|
||||
@@ -100,7 +116,8 @@ def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int):
|
||||
dataset=pipeline.datamanager.train_dataset,
|
||||
train_state="completed",
|
||||
)
|
||||
viewer_state.viser_server.set_training_state("completed")
|
||||
if isinstance(viewer_state, ViewerState):
|
||||
viewer_state.viser_server.set_training_state("completed")
|
||||
viewer_state.update_scene(step=step)
|
||||
while True:
|
||||
time.sleep(0.01)
|
||||
|
||||
@@ -243,6 +243,8 @@ class RenderStateMachine(threading.Thread):
|
||||
image_height: the maximum image height that can be rendered in the time budget
|
||||
image_width: the maximum image width that can be rendered in the time budget
|
||||
"""
|
||||
if aspect_ratio == 0:
|
||||
aspect_ratio = 0.001
|
||||
max_res = self.viewer.control_panel.max_res
|
||||
if self.state == "high":
|
||||
# high res is always static
|
||||
|
||||
@@ -118,8 +118,8 @@ class Viewer:
|
||||
def _(_: viser.CameraHandle) -> None:
|
||||
R = vtf.SO3(wxyz=client.camera.wxyz)
|
||||
R = R @ vtf.SO3.from_x_radians(np.pi)
|
||||
R = torch.tensor(R.as_matrix())
|
||||
pos = torch.tensor(client.camera.position, dtype=torch.float64) / VISER_NERFSTUDIO_SCALE_RATIO
|
||||
R = torch.tensor(R.as_matrix(), dtype=torch.float32)
|
||||
pos = torch.tensor(client.camera.position, dtype=torch.float32) / VISER_NERFSTUDIO_SCALE_RATIO
|
||||
c2w = torch.concatenate([R, pos[:, None]], dim=1)
|
||||
self.camera_state = CameraState(fov=client.camera.fov, aspect=client.camera.aspect, c2w=c2w)
|
||||
self.render_statemachine.action(RenderAction("move", self.camera_state))
|
||||
|
||||
Reference in new issue
Block a user