diff --git a/nerfstudio/cameras/utils.py b/nerfstudio/cameras/utils.py index 61a77039e..3913df6ab 100644 --- a/nerfstudio/cameras/utils.py +++ b/nerfstudio/cameras/utils.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# pylint: skip-file """ Camera transformation helper code. """ import math -from turtle import width from typing import List, Literal, Optional, Tuple import numpy as np @@ -118,7 +116,7 @@ def quaternion_slerp(quat0, quat1, fraction: float, spin: int = 0, shortestpath: raise ValueError("Input quaternions invalid.") if fraction == 0.0: return q0 - elif fraction == 1.0: + if fraction == 1.0: return q1 d = np.dot(q0, q1) if abs(abs(d) - 1.0) < _EPS: @@ -159,7 +157,7 @@ def quaternion_matrix(quaternion) -> np.ndarray: ) -def get_interpolated_poses(poseA, poseB, steps: int = 10) -> List[float]: +def get_interpolated_poses(pose_a, pose_b, steps: int = 10) -> List[float]: """Return interpolation of poses with specified number of steps. Args: poseA: first pose @@ -167,23 +165,23 @@ def get_interpolated_poses(poseA, poseB, steps: int = 10) -> List[float]: steps: number of steps the interpolated pose path should contain """ - quatA = quaternion_from_matrix(poseA[:3, :3]) - quatB = quaternion_from_matrix(poseB[:3, :3]) + quat_a = quaternion_from_matrix(pose_a[:3, :3]) + quat_b = quaternion_from_matrix(pose_b[:3, :3]) ts = np.linspace(0, 1, steps) - quats = [quaternion_slerp(quatA, quatB, t) for t in ts] - trans = [(1 - t) * poseA[:3, 3] + t * poseB[:3, 3] for t in ts] + quats = [quaternion_slerp(quat_a, quat_b, t) for t in ts] + trans = [(1 - t) * pose_a[:3, 3] + t * pose_b[:3, 3] for t in ts] - posesAB = [] + poses_ab = [] for quat, tran in zip(quats, trans): pose = np.identity(4) pose[:3, :3] = quaternion_matrix(quat)[:3, :3] pose[:3, 3] = tran - posesAB.append(pose) - return posesAB + poses_ab.append(pose) + return poses_ab -def get_interpolated_K(KA, KB, steps: int = 10) -> TensorType[3, 4]: +def get_interpolated_k(k_a, k_b, steps: int = 10) -> TensorType[3, 4]: """ Returns interpolated path between two camera poses with specified number of steps. @@ -195,8 +193,8 @@ def get_interpolated_K(KA, KB, steps: int = 10) -> TensorType[3, 4]: Ks = [] ts = np.linspace(0, 1, steps) for t in ts: - newK = KA * (1.0 - t) + KB * t - Ks.append(newK) + new_k = k_a * (1.0 - t) + k_b * t + Ks.append(new_k) return Ks @@ -218,31 +216,14 @@ def get_interpolated_poses_many( traj = [] Ks = [] for idx in range(poses.shape[0] - 1): - poseA = poses[idx] - poseB = poses[idx + 1] - posesAB = get_interpolated_poses(poseA, poseB, steps=steps_per_transition) - traj += posesAB - Ks += get_interpolated_K(Ks[idx], Ks[idx + 1], steps_per_transition) + pose_a = poses[idx] + pose_b = poses[idx + 1] + poses_ab = get_interpolated_poses(pose_a, pose_b, steps=steps_per_transition) + traj += poses_ab + Ks += get_interpolated_k(Ks[idx], Ks[idx + 1], steps_per_transition) return torch.stack(traj, dim=0), torch.stack(Ks, dim=0) -def get_swirl_poses(pose): - """ - Returns a swirl around the camera poses. - # TODO(ethan): add parameters and make this work + add docstrings + return type - """ - N_frames = 30 * 4 - dx = np.linspace(0, 0.03, N_frames) - dy = np.linspace(0, -0.1, N_frames) - dz = np.linspace(0, 0.5, N_frames) - poses = np.tile(pose, (N_frames, 1, 1)) - for i in range(N_frames): - poses[i, 0, 3] += dx[i] - poses[i, 1, 3] += dy[i] - poses[i, 2, 3] += dz[i] - return poses - - def normalize(x) -> TensorType[...]: """Returns a normalized vector.""" return x / torch.linalg.norm(x) @@ -418,7 +399,7 @@ def rotation_matrix(a: TensorType[3], b: TensorType[3]) -> TensorType[3, 3]: # If vectors are exactly opposite, we add a little noise to one of them if c < -1 + 1e-8: eps = (torch.rand(3) - 0.5) * 0.01 - return rotmat(a + eps, b) + return rotation_matrix(a + eps, b) s = torch.linalg.norm(v) skew_sym_mat = torch.Tensor( [ diff --git a/nerfstudio/models/modules/ray_losses.py b/nerfstudio/models/modules/ray_losses.py index 20d38ea28..0a7068185 100644 --- a/nerfstudio/models/modules/ray_losses.py +++ b/nerfstudio/models/modules/ray_losses.py @@ -50,6 +50,7 @@ def distortion_loss( starts = ray_samples.spacing_starts ends = ray_samples.spacing_ends + assert starts is not None and ends is not None, "Ray samples must have spacing starts and ends" midpoints = (starts + ends) / 2.0 # (..., num_samples, 1) loss = ( diff --git a/nerfstudio/models/modules/ray_sampler.py b/nerfstudio/models/modules/ray_sampler.py index 3137feda8..cfd9e88ea 100644 --- a/nerfstudio/models/modules/ray_sampler.py +++ b/nerfstudio/models/modules/ray_sampler.py @@ -502,6 +502,9 @@ class ProposalNetworkSampler(Sampler): self.num_nerf_samples_per_ray = num_nerf_samples_per_ray self.num_proposal_network_iterations = num_proposal_network_iterations + if self.num_proposal_network_iterations < 1: + raise ValueError("num_proposal_network_iterations must be >= 1") + # samplers self.initial_sampler = UniformLinDispPiecewiseSampler(single_jitter=single_jitter) self.pdf_sampler = PDFSampler(include_original=False) @@ -524,6 +527,8 @@ class ProposalNetworkSampler(Sampler): ray_samples_list = [] n = self.num_proposal_network_iterations + weights = None + ray_samples = None for i_level in range(n + 1): is_prop = i_level < n num_samples = self.num_proposal_samples_per_ray[i_level] if is_prop else self.num_nerf_samples_per_ray @@ -533,6 +538,7 @@ class ProposalNetworkSampler(Sampler): else: # PDF sampling based on the last samples and their weights # Perform annealing to the weights. This will be a no-op if self._anneal is 1.0. + assert weights is not None annealed_weights = torch.pow(weights, self._anneal) ray_samples = self.pdf_sampler(ray_bundle, ray_samples, annealed_weights, num_samples=num_samples) if is_prop: @@ -541,4 +547,5 @@ class ProposalNetworkSampler(Sampler): weights_list.append(weights) # (num_rays, num_samples) ray_samples_list.append(ray_samples) + assert ray_samples is not None return ray_samples, weights_list, ray_samples_list diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index cf4730dc3..19b7513f0 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -214,6 +214,7 @@ class NerfactoModel(Model): loss_dict["interlevel_loss"] = self.config.interlevel_loss_mult * interlevel_loss( outputs["weights_list"], outputs["ray_samples_list"] ) + assert metrics_dict is not None and "distortion" in metrics_dict loss_dict["distortion_loss"] = self.config.distortion_loss_mult * metrics_dict["distortion"] return loss_dict