diff --git a/nerfstudio/cameras/camera_optimizers.py b/nerfstudio/cameras/camera_optimizers.py index cd6611aab..44971683e 100644 --- a/nerfstudio/cameras/camera_optimizers.py +++ b/nerfstudio/cameras/camera_optimizers.py @@ -30,7 +30,7 @@ from typing_extensions import assert_never from nerfstudio.cameras.lie_groups import exp_map_SE3, exp_map_SO3xR3 from nerfstudio.configs.base_config import InstantiateConfig -from nerfstudio.engine.optimizers import AdamOptimizerConfig +from nerfstudio.engine.optimizers import AdamOptimizerConfig, OptimizerConfig from nerfstudio.engine.schedulers import ( ExponentialDecaySchedulerConfig, SchedulerConfig, @@ -53,7 +53,7 @@ class CameraOptimizerConfig(InstantiateConfig): orientation_noise_std: float = 0.0 """Noise to add to initial orientations. Useful for debugging.""" - optimizer: AdamOptimizerConfig = AdamOptimizerConfig(lr=6e-4, eps=1e-15) + optimizer: OptimizerConfig = AdamOptimizerConfig(lr=6e-4, eps=1e-15) """ADAM parameters for camera optimization.""" scheduler: SchedulerConfig = ExponentialDecaySchedulerConfig(max_steps=10000) diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 457b78f7b..6fdf8e1b5 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -423,7 +423,7 @@ class Cameras(TensorDataclass): if coords is None: index_dim = camera_indices.shape[-1] index = camera_indices.reshape(-1, index_dim)[0] - coords: torch.Tensor = cameras.get_image_coords(index=tuple(index)) # (h, w, 2) + coords = cameras.get_image_coords(index=tuple(index)) # (h, w, 2) coords = coords.reshape(coords.shape[:2] + (1,) * len(camera_indices.shape[:-1]) + (2,)) # (h, w, 1..., 2) coords = coords.expand(coords.shape[:2] + camera_indices.shape[:-1] + (2,)) # (h, w, num_rays, 2) camera_opt_to_camera = ( # (h, w, num_rays, 3, 4) or None @@ -766,7 +766,7 @@ class Cameras(TensorDataclass): image_uint8 = torchvision.transforms.functional.resize(image_uint8, max_size) # type: ignore image_uint8 = image_uint8.permute(1, 2, 0) image_uint8 = image_uint8.cpu().numpy() - data = cv2.imencode(".jpg", image_uint8)[1].tobytes() + data = cv2.imencode(".jpg", image_uint8)[1].tobytes() # type: ignore json_["image"] = str("data:image/jpeg;base64," + base64.b64encode(data).decode("ascii")) return json_ diff --git a/nerfstudio/cameras/rays.py b/nerfstudio/cameras/rays.py index 18bcabd14..113622d53 100644 --- a/nerfstudio/cameras/rays.py +++ b/nerfstudio/cameras/rays.py @@ -16,8 +16,8 @@ Some ray datastructures. """ import random -from dataclasses import dataclass -from typing import Callable, Dict, Optional, Tuple, Union +from dataclasses import dataclass, field +from typing import Callable, Dict, Literal, Optional, Tuple, Union, overload import torch from jaxtyping import Float, Int, Shaped @@ -149,10 +149,27 @@ class RaySamples(TensorDataclass): return weights + @overload + @staticmethod + def get_weights_and_transmittance_from_alphas( + alphas: Float[Tensor, "*batch num_samples 1"], weights_only: Literal[False] = False + ) -> Tuple[Float[Tensor, "*batch num_samples 1"], Float[Tensor, "*batch num_samples 1"]]: + ... + + @overload + @staticmethod + def get_weights_and_transmittance_from_alphas( + alphas: Float[Tensor, "*batch num_samples 1"], weights_only: Literal[True] + ) -> Tuple[Float[Tensor, "*batch num_samples 1"], Float[Tensor, "*batch num_samples 1"]]: + ... + @staticmethod def get_weights_and_transmittance_from_alphas( alphas: Float[Tensor, "*batch num_samples 1"], weights_only: bool = False - ) -> Tuple[Float[Tensor, "*batch num_samples 1"], Float[Tensor, "*batch num_samples 1"]]: + ) -> Union[ + Float[Tensor, "*batch num_samples 1"], + Tuple[Float[Tensor, "*batch num_samples 1"], Float[Tensor, "*batch num_samples 1"]], + ]: """Return weights based on predicted alphas Args: alphas: Predicted alphas (maybe from sdf) for samples along ray @@ -188,7 +205,7 @@ class RayBundle(TensorDataclass): """Distance along ray to start sampling""" fars: Optional[Float[Tensor, "*batch 1"]] = None """Rays Distance along ray to stop sampling""" - metadata: Optional[Dict[str, Shaped[Tensor, "num_rays latent_dims"]]] = None + metadata: Dict[str, Shaped[Tensor, "num_rays latent_dims"]] = field(default_factory=dict) """Additional metadata or data needed for interpolation, will mimic shape of rays""" times: Optional[Float[Tensor, "*batch 1"]] = None """Times at which rays are sampled""" diff --git a/nerfstudio/configs/method_configs.py b/nerfstudio/configs/method_configs.py index 494e47cb3..8fe4ac57b 100644 --- a/nerfstudio/configs/method_configs.py +++ b/nerfstudio/configs/method_configs.py @@ -132,7 +132,8 @@ method_configs["nerfacto-big"] = TrainerConfig( train_num_rays_per_batch=4096, eval_num_rays_per_batch=4096, camera_optimizer=CameraOptimizerConfig( - mode="SO3xR3", optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3) + mode="SO3xR3", + optimizer=RAdamOptimizerConfig(lr=6e-4, eps=1e-8, weight_decay=1e-3), ), ), model=NerfactoModelConfig( diff --git a/nerfstudio/data/datamanagers/base_datamanager.py b/nerfstudio/data/datamanagers/base_datamanager.py index 869434197..4c2db5843 100644 --- a/nerfstudio/data/datamanagers/base_datamanager.py +++ b/nerfstudio/data/datamanagers/base_datamanager.py @@ -22,7 +22,18 @@ import functools from abc import abstractmethod from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Dict, Generic, List, Literal, Optional, Tuple, Type, Union +from typing import ( + Any, + Callable, + Dict, + Generic, + List, + Literal, + Optional, + Tuple, + Type, + Union, +) import torch from torch import nn @@ -72,7 +83,7 @@ def variable_res_collate(batch: List[Dict]) -> Dict: if mask is not None: masks.append(mask) - new_batch: dict = nerfstudio_collate(batch) + new_batch = nerfstudio_collate(batch) new_batch["image"] = images if masks: new_batch["mask"] = masks @@ -314,7 +325,7 @@ class VanillaDataManagerConfig(DataManagerConfig): camera_optimizer: CameraOptimizerConfig = CameraOptimizerConfig() """Specifies the camera pose optimizer used during training. Helpful if poses are noisy, such as for data from Record3D.""" - collate_fn = staticmethod(nerfstudio_collate) + collate_fn: Callable[[Any], Any] = staticmethod(nerfstudio_collate) """Specifies the collate function to use for the train and eval dataloaders.""" camera_res_scale_factor: float = 1.0 """The scale factor for scaling spatial data such as images, mask, semantics @@ -356,7 +367,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs local_rank: int = 0, **kwargs, # pylint: disable=unused-argument ): - self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", TDataset.__default__) + self.dataset_type: Type[TDataset] = kwargs.get("_dataset_type", getattr(TDataset, "__default__")) self.config = config self.device = device self.world_size = world_size @@ -486,6 +497,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs self.train_count += 1 image_batch = next(self.iter_train_image_dataloader) assert self.train_pixel_sampler is not None + assert isinstance(image_batch, dict) batch = self.train_pixel_sampler.sample(image_batch) ray_indices = batch["indices"] ray_bundle = self.train_ray_generator(ray_indices) @@ -496,6 +508,7 @@ class VanillaDataManager(DataManager, Generic[TDataset]): # pylint: disable=abs self.eval_count += 1 image_batch = next(self.iter_eval_image_dataloader) assert self.eval_pixel_sampler is not None + assert isinstance(image_batch, dict) batch = self.eval_pixel_sampler.sample(image_batch) ray_indices = batch["indices"] ray_bundle = self.eval_ray_generator(ray_indices) diff --git a/nerfstudio/data/dataparsers/arkitscenes_dataparser.py b/nerfstudio/data/dataparsers/arkitscenes_dataparser.py index ccfaa0c16..290bd6810 100644 --- a/nerfstudio/data/dataparsers/arkitscenes_dataparser.py +++ b/nerfstudio/data/dataparsers/arkitscenes_dataparser.py @@ -50,7 +50,7 @@ def traj_string_to_matrix(traj_string: str): ts = tokens[0] # Rotation in angle axis angle_axis = [float(tokens[1]), float(tokens[2]), float(tokens[3])] - r_w_to_p, _ = cv2.Rodrigues(np.asarray(angle_axis)) + r_w_to_p, _ = cv2.Rodrigues(np.asarray(angle_axis)) # type: ignore # Translation t_w_to_p = np.asarray([float(tokens[4]), float(tokens[5]), float(tokens[6])]) extrinsics = np.eye(4, 4) @@ -213,6 +213,7 @@ class ARKitScenes(DataParser): @staticmethod def _get_pose(frame_id: str, poses_from_traj: dict): + frame_pose = None if str(frame_id) in poses_from_traj: frame_pose = np.array(poses_from_traj[str(frame_id)]) else: @@ -220,6 +221,7 @@ class ARKitScenes(DataParser): if abs(float(frame_id) - float(my_key)) < 0.005: frame_pose = np.array(poses_from_traj[str(my_key)]) + assert frame_pose is not None frame_pose[0:3, 1:3] *= -1 frame_pose = frame_pose[np.array([1, 0, 2, 3]), :] frame_pose[2, :] *= -1 diff --git a/nerfstudio/data/dataparsers/dycheck_dataparser.py b/nerfstudio/data/dataparsers/dycheck_dataparser.py index fc6c0ae51..1d7319023 100644 --- a/nerfstudio/data/dataparsers/dycheck_dataparser.py +++ b/nerfstudio/data/dataparsers/dycheck_dataparser.py @@ -52,7 +52,7 @@ def downscale(img, scale: int) -> np.ndarray: if height % scale > 0 or width % scale > 0: raise ValueError(f"Image shape ({height},{width}) must be divisible by the" f" scale ({scale}).") out_height, out_width = height // scale, width // scale - resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) + resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) # type: ignore return resized @@ -70,11 +70,11 @@ def upscale(img, scale: int) -> np.ndarray: return img height, width = img.shape[:2] out_height, out_width = height * scale, width * scale - resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) + resized = cv2.resize(img, (out_width, out_height), cv2.INTER_AREA) # type: ignore return resized -def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarray: +def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarray: # type: ignore """Function from DyCheck's repo. Rescale an image. Args: @@ -106,7 +106,7 @@ def rescale(img, scale_factor: float, interpolation=cv2.INTER_AREA) -> np.ndarra out_width = math.ceil(width * scale_factor) out_width -= out_width % 2 - return cv2.resize(img, (out_width, out_height), interpolation) + return cv2.resize(img, (out_width, out_height), interpolation) # type: ignore def _load_scene_info(data_dir: Path) -> Tuple[np.ndarray, float, float, float]: @@ -242,7 +242,7 @@ class Dycheck(DataParser): frame_names = np.array(split_dict["frame_names"])[[0]] time_ids = np.array(split_dict["time_ids"])[[0]] - image_filenames, depth_filenames, cams = self.process_frames(frame_names, time_ids) + image_filenames, depth_filenames, cams = self.process_frames(frame_names.tolist(), time_ids) scene_box = SceneBox( aabb=torch.tensor( @@ -316,13 +316,13 @@ class Dycheck(DataParser): d = self.config.downscale_factor if not image_filenames[0].exists(): CONSOLE.print(f"downscale factor {d}x not exist, converting") - ori_h, ori_w = cv2.imread(str(self.data / f"rgb/1x/{frame_names[0]}.png")).shape[:2] + ori_h, ori_w = cv2.imread(str(self.data / f"rgb/1x/{frame_names[0]}.png")).shape[:2] # type: ignore (self.data / f"rgb/{d}x").mkdir(exist_ok=True) h, w = ori_h // d, ori_w // d for frame in frame_names: - cv2.imwrite( + cv2.imwrite( # type: ignore str(self.data / f"rgb/{d}x/{frame}.png"), - cv2.resize(cv2.imread(str(self.data / f"rgb/1x/{frame}.png")), [h, w]), + cv2.resize(cv2.imread(str(self.data / f"rgb/1x/{frame}.png")), [h, w]), # type: ignore ) CONSOLE.print("finished") @@ -331,8 +331,8 @@ class Dycheck(DataParser): (self.data / f"processed_depth/{d}x").mkdir(exist_ok=True, parents=True) for idx, frame in enumerate(frame_names): depth = np.load(self.data / f"depth/1x/{frame}.npy") - mask = rescale((depth != 0).astype(np.uint8) * 255, 1 / d, cv2.INTER_AREA) - depth = rescale(depth, 1 / d, cv2.INTER_AREA) + mask = rescale((depth != 0).astype(np.uint8) * 255, 1 / d, cv2.INTER_AREA) # type: ignore + depth = rescale(depth, 1 / d, cv2.INTER_AREA) # type: ignore depth[mask != 255] = 0 depth = _rescale_depth(depth, cams[idx]) np.save(str(self.data / f"processed_depth/{d}x/{frame}.npy"), depth) diff --git a/nerfstudio/data/dataparsers/minimal_dataparser.py b/nerfstudio/data/dataparsers/minimal_dataparser.py index 8f410ddc0..1aa40c12a 100644 --- a/nerfstudio/data/dataparsers/minimal_dataparser.py +++ b/nerfstudio/data/dataparsers/minimal_dataparser.py @@ -57,7 +57,6 @@ class MinimalDataParser(DataParser): if "mask_filenames" in data.keys(): mask_filenames = [filepath.parent / path for path in data["mask_filenames"].tolist()] - metadata = None if "semantics" in data.keys(): semantics = data["semantics"].item() metadata = { @@ -68,6 +67,8 @@ class MinimalDataParser(DataParser): mask_classes=semantics["mask_classes"].tolist(), ) } + else: + metadata = {} scene_box_aabb = torch.from_numpy(data["scene_box"]) scene_box = SceneBox(aabb=scene_box_aabb) @@ -90,7 +91,7 @@ class MinimalDataParser(DataParser): ) applied_scale = 1.0 - applied_transform = np.eye(4, dtype=np.float32)[:3, :] + applied_transform = torch.eye(4, dtype=torch.float32)[:3, :] if "applied_scale" in data.keys(): applied_scale = float(data["applied_scale"]) if "applied_transform" in data.keys(): diff --git a/nerfstudio/data/dataparsers/nuscenes_dataparser.py b/nerfstudio/data/dataparsers/nuscenes_dataparser.py index f45d63bc6..056e11ec5 100644 --- a/nerfstudio/data/dataparsers/nuscenes_dataparser.py +++ b/nerfstudio/data/dataparsers/nuscenes_dataparser.py @@ -87,7 +87,11 @@ class NuScenes(DataParser): def _generate_dataparser_outputs(self, split="train"): # pylint: disable=too-many-statements - nusc = NuScenesDatabase(version=self.config.version, dataroot=self.config.data_dir, verbose=self.config.verbose) + nusc = NuScenesDatabase( + version=self.config.version, + dataroot=str(self.config.data_dir.absolute()), + verbose=self.config.verbose, + ) cameras = ["CAM_" + camera for camera in self.config.cameras] assert ( diff --git a/nerfstudio/data/dataparsers/scannet_dataparser.py b/nerfstudio/data/dataparsers/scannet_dataparser.py index f1a9d9706..578815c3c 100644 --- a/nerfstudio/data/dataparsers/scannet_dataparser.py +++ b/nerfstudio/data/dataparsers/scannet_dataparser.py @@ -81,7 +81,7 @@ class ScanNet(DataParser): depth_dir_sorted = list(sorted(depth_dir.iterdir(), key=lambda x: int(x.name.split(".")[0]))) pose_dir_sorted = list(sorted(pose_dir.iterdir(), key=lambda x: int(x.name.split(".")[0]))) - first_img = cv2.imread(img_dir_sorted[0].as_posix()) + first_img = cv2.imread(str(img_dir_sorted[0].absolute())) # type: ignore h, w, _ = first_img.shape image_filenames, depth_filenames, intrinsics, poses = [], [], [], [] diff --git a/nerfstudio/data/dataparsers/sitcoms3d_dataparser.py b/nerfstudio/data/dataparsers/sitcoms3d_dataparser.py index 1dc24fc22..1d90a49d9 100644 --- a/nerfstudio/data/dataparsers/sitcoms3d_dataparser.py +++ b/nerfstudio/data/dataparsers/sitcoms3d_dataparser.py @@ -111,11 +111,12 @@ class Sitcoms3D(DataParser): lengths = scene_box.aabb[1] - scene_box.aabb[0] longest_dim = torch.argmax(lengths) longest_length = lengths[longest_dim] - scale = scene_scale / longest_length + scale = scene_scale / longest_length.item() scene_box.aabb = scene_box.aabb * scale # box camera_to_worlds[..., 3] *= scale # cameras # --- semantics --- + semantics = None if self.config.include_semantics: empty_path = Path() replace_this_path = str(empty_path / images_folder / empty_path) diff --git a/nerfstudio/data/datasets/base_dataset.py b/nerfstudio/data/datasets/base_dataset.py index f3245c675..9cc94c941 100644 --- a/nerfstudio/data/datasets/base_dataset.py +++ b/nerfstudio/data/datasets/base_dataset.py @@ -44,7 +44,6 @@ class InputDataset(Dataset): def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0): super().__init__() self._dataparser_outputs = dataparser_outputs - self.has_masks = dataparser_outputs.mask_filenames is not None self.scale_factor = scale_factor self.scene_box = deepcopy(dataparser_outputs.scene_box) self.metadata = deepcopy(dataparser_outputs.metadata) @@ -95,9 +94,8 @@ class InputDataset(Dataset): image_idx: The image index in the dataset. """ image = self.get_image(image_idx) - data = {"image_idx": image_idx} - data["image"] = image - if self.has_masks: + data = {"image_idx": image_idx, "image": image} + if self._dataparser_outputs.mask_filenames is not None: mask_filepath = self._dataparser_outputs.mask_filenames[image_idx] data["mask"] = get_image_mask_tensor_from_path(filepath=mask_filepath, scale_factor=self.scale_factor) assert ( diff --git a/nerfstudio/data/datasets/sdf_dataset.py b/nerfstudio/data/datasets/sdf_dataset.py index eea7c323c..37d541220 100644 --- a/nerfstudio/data/datasets/sdf_dataset.py +++ b/nerfstudio/data/datasets/sdf_dataset.py @@ -81,7 +81,7 @@ class SDFDataset(InputDataset): normal = normal * 2.0 - 1.0 # omnidata output is normalized so we convert it back to normal here normal = torch.from_numpy(normal).float() - rot = camtoworld[:3, :3] + rot = torch.from_numpy(camtoworld[:3, :3]) normal_map = normal.reshape(3, -1) normal_map = torch.nn.functional.normalize(normal_map, p=2, dim=0) diff --git a/nerfstudio/data/utils/colmap_parsing_utils.py b/nerfstudio/data/utils/colmap_parsing_utils.py index 2b7c48cc5..aea19e144 100644 --- a/nerfstudio/data/utils/colmap_parsing_utils.py +++ b/nerfstudio/data/utils/colmap_parsing_utils.py @@ -479,7 +479,7 @@ def rotmat2qvec(R): Rxx, Ryx, Rzx, Rxy, Ryy, Rzy, Rxz, Ryz, Rzz = R.flat K = ( np.array( - [ + [ # type: ignore [Rxx - Ryy - Rzz, 0, 0, 0], [Ryx + Rxy, Ryy - Rxx - Rzz, 0, 0], [Rzx + Rxz, Rzy + Ryz, Rzz - Rxx - Ryy, 0], diff --git a/nerfstudio/data/utils/data_utils.py b/nerfstudio/data/utils/data_utils.py index 391f424a2..0f0e911d1 100644 --- a/nerfstudio/data/utils/data_utils.py +++ b/nerfstudio/data/utils/data_utils.py @@ -45,7 +45,7 @@ def get_semantics_and_mask_tensors_from_path( If no mask is required - use mask_indices = [] """ if isinstance(mask_indices, List): - mask_indices = torch.tensor(mask_indices, dtype="int64").view(1, 1, -1) + mask_indices = torch.tensor(mask_indices, dtype=torch.int64).view(1, 1, -1) pil_image = Image.open(filepath) if scale_factor != 1.0: width, height = pil_image.size @@ -61,7 +61,7 @@ def get_depth_image_from_path( height: int, width: int, scale_factor: float, - interpolation: int = cv2.INTER_NEAREST, + interpolation: int = cv2.INTER_NEAREST, # type: ignore ) -> torch.Tensor: """Loads, rescales and resizes depth images. Filepath points to a 16-bit or 32-bit depth image, or a numpy array `*.npy`. @@ -78,9 +78,9 @@ def get_depth_image_from_path( """ if filepath.suffix == ".npy": image = np.load(filepath) * scale_factor - image = cv2.resize(image, (width, height), interpolation=interpolation) + image = cv2.resize(image, (width, height), interpolation=interpolation) # type: ignore else: - image = cv2.imread(str(filepath.absolute()), cv2.IMREAD_ANYDEPTH) + image = cv2.imread(str(filepath.absolute()), cv2.IMREAD_ANYDEPTH) # type: ignore image = image.astype(np.float64) * scale_factor - image = cv2.resize(image, (width, height), interpolation=interpolation) + image = cv2.resize(image, (width, height), interpolation=interpolation) # type: ignore return torch.from_numpy(image[:, :, np.newaxis]) diff --git a/nerfstudio/data/utils/dataloaders.py b/nerfstudio/data/utils/dataloaders.py index 5b9bf9dbc..b3b4aea4a 100644 --- a/nerfstudio/data/utils/dataloaders.py +++ b/nerfstudio/data/utils/dataloaders.py @@ -21,7 +21,7 @@ import concurrent.futures import multiprocessing import random from abc import abstractmethod -from typing import Dict, Optional, Tuple, Union +from typing import Dict, Optional, Sized, Tuple, Union import torch from rich.progress import track @@ -58,6 +58,8 @@ class CacheDataloader(DataLoader): **kwargs, ): self.dataset = dataset + assert isinstance(self.dataset, Sized) + super().__init__(dataset=dataset, **kwargs) # This will set self.dataset self.num_times_to_repeat_images = num_times_to_repeat_images self.cache_all_images = (num_images_to_sample_from == -1) or (num_images_to_sample_from >= len(self.dataset)) @@ -93,6 +95,7 @@ class CacheDataloader(DataLoader): def _get_batch_list(self): """Returns a list of batches from the dataset attribute.""" + assert isinstance(self.dataset, Sized) indices = random.sample(range(len(self.dataset)), k=self.num_images_to_sample_from) batch_list = [] results = [] @@ -183,6 +186,7 @@ class EvalDataloader(DataLoader): ray_bundle = self.cameras.generate_rays(camera_indices=image_idx, keep_shape=True) batch = self.input_dataset[image_idx] batch = get_dict_to_torch(batch, device=self.device, exclude=["image"]) + assert isinstance(batch, dict) return ray_bundle, batch diff --git a/nerfstudio/data/utils/nerfstudio_collate.py b/nerfstudio/data/utils/nerfstudio_collate.py index 8fbe1c1f6..5dfcee15f 100644 --- a/nerfstudio/data/utils/nerfstudio_collate.py +++ b/nerfstudio/data/utils/nerfstudio_collate.py @@ -19,7 +19,7 @@ Custom collate function that includes cases for nerfstudio types. import collections import collections.abc import re -from typing import Callable, Dict, Union +from typing import Any, Callable, Dict, Union import torch import torch.utils.data @@ -33,9 +33,9 @@ NERFSTUDIO_COLLATE_ERR_MSG_FORMAT = ( np_str_obj_array_pattern = re.compile(r"[SaUO]") -def nerfstudio_collate( - batch, extra_mappings: Union[Dict[type, Callable], None] = None -): # pylint: disable=too-many-return-statements +def nerfstudio_collate( # pylint: disable=too-many-return-statements + batch: Any, extra_mappings: Union[Dict[type, Callable], None] = None +) -> Any: r""" This is the default pytorch collate function, but with support for nerfstudio types. All documentation below is copied straight over from pytorch's default_collate function, python version 3.8.13, diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index ca5e07969..1bceb60ea 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -24,7 +24,7 @@ import time from dataclasses import dataclass, field from pathlib import Path from threading import Lock -from typing import Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Dict, List, Literal, Optional, Tuple, Type, cast import torch from rich import box, style @@ -54,7 +54,7 @@ from nerfstudio.viewer.server.viewer_state import ViewerState TRAIN_INTERATION_OUTPUT = Tuple[ # pylint: disable=invalid-name torch.Tensor, Dict[str, torch.Tensor], Dict[str, torch.Tensor] ] -TORCH_DEVICE = Union[torch.device, str] # pylint: disable=invalid-name +TORCH_DEVICE = str # pylint: disable=invalid-name @dataclass @@ -374,8 +374,8 @@ class Trainer: def _load_checkpoint(self) -> None: """Helper function to load pipeline and optimizer from prespecified checkpoint""" - load_dir: Path = self.config.load_dir - load_checkpoint: Path = self.config.load_checkpoint + load_dir = self.config.load_dir + load_checkpoint = self.config.load_checkpoint if load_dir is not None: load_step = self.config.load_step if load_step is None: @@ -459,7 +459,7 @@ class Trainer: metrics_dict[f"Gradients/{tag}"] = grad total_grad += grad - metrics_dict["Gradients/Total"] = total_grad + metrics_dict["Gradients/Total"] = cast(torch.Tensor, total_grad) scale = self.grad_scaler.get_scale() self.grad_scaler.update() diff --git a/nerfstudio/exporter/exporter_utils.py b/nerfstudio/exporter/exporter_utils.py index 3e1088eab..a234f8490 100644 --- a/nerfstudio/exporter/exporter_utils.py +++ b/nerfstudio/exporter/exporter_utils.py @@ -58,7 +58,7 @@ class Mesh: """Colors of the mesh.""" -def get_mesh_from_pymeshlab_mesh(mesh: pymeshlab.Mesh) -> Mesh: +def get_mesh_from_pymeshlab_mesh(mesh: pymeshlab.Mesh) -> Mesh: # type: ignore """Get a Mesh from a pymeshlab mesh. See https://pymeshlab.readthedocs.io/en/0.1.5/classes/mesh.html for details. """ @@ -72,7 +72,7 @@ def get_mesh_from_pymeshlab_mesh(mesh: pymeshlab.Mesh) -> Mesh: def get_mesh_from_filename(filename: str, target_num_faces: Optional[int] = None) -> Mesh: """Get a Mesh from a filename.""" - ms = pymeshlab.MeshSet() + ms = pymeshlab.MeshSet() # type: ignore ms.load_new_mesh(filename) if target_num_faces is not None: CONSOLE.print("Running meshing decimation with quadric edge collapse") @@ -128,6 +128,8 @@ def generate_point_cloud( with progress as progress_bar: task = progress_bar.add_task("Generating Point Cloud", total=num_points) while not progress_bar.finished: + normal = None + with torch.no_grad(): ray_bundle, _ = pipeline.datamanager.next_train(0) outputs = pipeline.model(ray_bundle) @@ -165,12 +167,12 @@ def generate_point_cloud( mask = torch.all(torch.concat([point > comp_l, point < comp_m], dim=-1), dim=-1) point = point[mask] rgb = rgb[mask] - if normal_output_name is not None: + if normal is not None: normal = normal[mask] points.append(point) rgbs.append(rgb) - if normal_output_name is not None: + if normal is not None: normals.append(normal) progress.advance(task, point.shape[0]) points = torch.cat(points, dim=0) diff --git a/nerfstudio/exporter/marching_cubes.py b/nerfstudio/exporter/marching_cubes.py index 7b38d619e..1a9292f18 100644 --- a/nerfstudio/exporter/marching_cubes.py +++ b/nerfstudio/exporter/marching_cubes.py @@ -17,7 +17,7 @@ This module implements the Marching Cubes algorithm for extracting isosurfaces """ -from typing import Callable, List, Tuple, Union +from typing import Callable, List, Optional, Tuple, Union import numpy as np import torch @@ -48,7 +48,7 @@ def create_point_pyramid(points: Float[Tensor, "3 height width depth"]) -> List[ return points_pyramid -def evaluate_sdf(sdf: Callable, points: Float[Tensor, "batch 3"]) -> Float[Tensor, "batch"]: +def evaluate_sdf(sdf: Callable[[Tensor], Tensor], points: Float[Tensor, "batch 3"]) -> Float[Tensor, "batch"]: """ Evaluate a signed distance function (SDF) for a batch of points. @@ -60,11 +60,10 @@ def evaluate_sdf(sdf: Callable, points: Float[Tensor, "batch 3"]) -> Float[Tenso Returns: A torch tensor with the SDF values evaluated at the given points. """ - z = [] + z: List[Tensor] = [] for _, pnts in enumerate(torch.split(points, 100000, dim=0)): z.append(sdf(pnts)) - z = torch.cat(z, axis=0) - return z + return torch.cat(z, dim=0) def evaluate_multiresolution_sdf( @@ -91,6 +90,7 @@ def evaluate_multiresolution_sdf( A torch tensor with the SDF values evaluated at the given points. """ mask = None + pts_sdf: Optional[Tensor] = None threshold = 2 * (x_max - x_min) / crop_n * 8 for pid, pts in enumerate(points_pyramid): coarse_n = pts.shape[-1] @@ -111,19 +111,24 @@ def evaluate_multiresolution_sdf( if pts_to_eval.shape[0] > 0: pts_sdf_eval = evaluate(pts_to_eval.contiguous()) - pts_sdf[mask] = pts_sdf_eval + assert pts_sdf is not None + pts_sdf[mask] = pts_sdf_eval # pylint: disable=unsupported-assignment-operation if pid < 3: # Update mask + assert pts_sdf is not None mask = torch.abs(pts_sdf) < threshold mask = mask.reshape(coarse_n, coarse_n, coarse_n)[None, None] mask = upsample(mask.float()).bool() pts_sdf = pts_sdf.reshape(coarse_n, coarse_n, coarse_n)[None, None] pts_sdf = upsample(pts_sdf) + assert pts_sdf is not None pts_sdf = pts_sdf.reshape(-1) threshold /= 2.0 + + assert pts_sdf is not None return pts_sdf @@ -232,7 +237,7 @@ def generate_mesh_with_multires_marching_cubes( if not (np.min(z) > isosurface_threshold or np.max(z) < isosurface_threshold): z = z.astype(np.float32) - verts, faces, normals, _ = measure.marching_cubes( + verts, faces, normals, _ = measure.marching_cubes( # type: ignore volume=z.reshape(crop_n, crop_n, crop_n), level=isosurface_threshold, spacing=( @@ -247,5 +252,5 @@ def generate_mesh_with_multires_marching_cubes( meshcrop = trimesh.Trimesh(verts, faces, normals) meshes.append(meshcrop) - combined_mesh: trimesh.Trimesh = trimesh.util.concatenate(meshes) + combined_mesh: trimesh.Trimesh = trimesh.util.concatenate(meshes) # type: ignore return combined_mesh diff --git a/nerfstudio/exporter/texture_utils.py b/nerfstudio/exporter/texture_utils.py index 3fde25985..72aaf5655 100644 --- a/nerfstudio/exporter/texture_utils.py +++ b/nerfstudio/exporter/texture_utils.py @@ -22,7 +22,7 @@ from __future__ import annotations import math from pathlib import Path -from typing import Literal, Optional, Tuple, Union +from typing import Literal, Tuple, Union import mediapy as media import numpy as np @@ -332,7 +332,7 @@ def export_textured_mesh( mesh: Mesh, pipeline: Pipeline, output_dir: Path, - px_per_uv_triangle: Optional[int] = None, + px_per_uv_triangle: int, unwrap_method: Literal["xatlas", "custom"] = "xatlas", raylen_method: Literal["edge", "none"] = "edge", num_pixels_per_side=1024, diff --git a/nerfstudio/exporter/tsdf_utils.py b/nerfstudio/exporter/tsdf_utils.py index ca99e80ad..884751073 100644 --- a/nerfstudio/exporter/tsdf_utils.py +++ b/nerfstudio/exporter/tsdf_utils.py @@ -83,7 +83,7 @@ class TSDF: def truncation(self) -> float: """Returns the truncation distance.""" # TODO: clean this up - truncation = self.voxel_size[0] * self.truncation_margin + truncation = self.voxel_size[0].item() * self.truncation_margin return truncation @staticmethod @@ -99,9 +99,9 @@ class TSDF: voxel_size = (aabb[1] - aabb[0]) / volume_dims # create the voxel coordinates - xdim = torch.arange(volume_dims[0]) - ydim = torch.arange(volume_dims[1]) - zdim = torch.arange(volume_dims[2]) + xdim = torch.arange(volume_dims[0].item()) + ydim = torch.arange(volume_dims[1].item()) + zdim = torch.arange(volume_dims[2].item()) grid = torch.stack(torch.meshgrid([xdim, ydim, zdim], indexing="ij"), dim=0) voxel_coords = origin.view(3, 1, 1, 1) + grid * voxel_size.view(3, 1, 1, 1) @@ -121,7 +121,11 @@ class TSDF: # run marching cubes on CPU tsdf_values_np = self.values.clamp(-1, 1).cpu().numpy() - vertices, faces, normals, _ = measure.marching_cubes(tsdf_values_np, level=0, allow_degenerate=False) + vertices, faces, normals, _ = measure.marching_cubes( # type: ignore + tsdf_values_np, + level=0, + allow_degenerate=False, + ) vertices_indices = np.round(vertices).astype(int) colors = self.colors[vertices_indices[:, 0], vertices_indices[:, 1], vertices_indices[:, 2]] @@ -145,6 +149,7 @@ class TSDF: mesh: The mesh to export. filename: The filename to export the mesh to. """ + assert mesh.colors is not None vertex_matrix = mesh.vertices.cpu().numpy().astype("float64") face_matrix = mesh.faces.cpu().numpy().astype("int32") v_normals_matrix = mesh.normals.cpu().numpy().astype("float64") @@ -153,14 +158,14 @@ class TSDF: v_color_matrix = np.concatenate([v_color_matrix, np.ones((v_color_matrix.shape[0], 1))], axis=-1) # create a new Mesh - m = pymeshlab.Mesh( + m = pymeshlab.Mesh( # type: ignore vertex_matrix=vertex_matrix, face_matrix=face_matrix, v_normals_matrix=v_normals_matrix, v_color_matrix=v_color_matrix, ) # create a new MeshSet - ms = pymeshlab.MeshSet() + ms = pymeshlab.MeshSet() # type: ignore # add the mesh to the MeshSet ms.add_mesh(m, "mesh") # save the current mesh @@ -230,6 +235,7 @@ class TSDF: ) # [batch, N, 1] sampled_depth = sampled_depth.squeeze(2) # [batch, 1, N] # colors + sampled_colors = None if color_images is not None: sampled_colors = F.grid_sample( input=color_images, grid=grid, mode="nearest", padding_mode="zeros", align_corners=False @@ -262,7 +268,7 @@ class TSDF: ) / total_weights self.weights[valid_points_i_shape] = torch.clamp(total_weights, max=1.0) - if color_images is not None: + if sampled_colors is not None: old_colors_i = self.colors[valid_points_i_shape] # [M, 3] new_colors_i = sampled_colors[i][:, valid_points_i.squeeze(0)].permute(1, 0) # [M, 3] self.colors[valid_points_i_shape] = ( diff --git a/nerfstudio/field_components/spatial_distortions.py b/nerfstudio/field_components/spatial_distortions.py index ec9fbac56..5e9aeab80 100644 --- a/nerfstudio/field_components/spatial_distortions.py +++ b/nerfstudio/field_components/spatial_distortions.py @@ -14,6 +14,7 @@ """Space distortions.""" +import abc from typing import Optional, Union import torch @@ -27,6 +28,7 @@ from nerfstudio.utils.math import Gaussians class SpatialDistortion(nn.Module): """Apply spatial distortions""" + @abc.abstractmethod def forward(self, positions: Union[Float[Tensor, "*bs 3"], Gaussians]) -> Union[Float[Tensor, "*bs 3"], Gaussians]: """ Args: diff --git a/nerfstudio/field_components/temporal_distortions.py b/nerfstudio/field_components/temporal_distortions.py index 7f3e4c3dd..468b2112b 100644 --- a/nerfstudio/field_components/temporal_distortions.py +++ b/nerfstudio/field_components/temporal_distortions.py @@ -14,6 +14,7 @@ """Space distortions which occur as a function of time.""" +import abc from enum import Enum from typing import Any, Dict, Optional, Tuple @@ -28,6 +29,7 @@ from nerfstudio.field_components.mlp import MLP class TemporalDistortion(nn.Module): """Apply spatial distortions as a function of time""" + @abc.abstractmethod def forward(self, positions: Float[Tensor, "*bs 3"], times: Optional[Float[Tensor, "1"]]) -> Float[Tensor, "*bs 3"]: """ Args: diff --git a/nerfstudio/model_components/losses.py b/nerfstudio/model_components/losses.py index a0cc6f22d..b2d492f68 100644 --- a/nerfstudio/model_components/losses.py +++ b/nerfstudio/model_components/losses.py @@ -16,7 +16,7 @@ Collection of Losses. """ from enum import Enum -from typing import Dict, Literal +from typing import Dict, Literal, Optional, TypeVar, cast import torch from jaxtyping import Bool, Float @@ -145,8 +145,8 @@ def distortion_loss(weights_list, ray_samples_list): def nerfstudio_distortion_loss( ray_samples: RaySamples, - densities: Float[Tensor, "*bs num_samples 1"] = None, - weights: Float[Tensor, "*bs num_samples 1"] = None, + densities: Optional[Float[Tensor, "*bs num_samples 1"]] = None, + weights: Optional[Float[Tensor, "*bs num_samples 1"]] = None, ) -> Float[Tensor, "*bs 1"]: """Ray based distortion loss proposed in MipNeRF-360. Returns distortion Loss. @@ -165,10 +165,12 @@ def nerfstudio_distortion_loss( """ if torch.is_tensor(densities): assert not torch.is_tensor(weights), "Cannot use both densities and weights" + assert densities is not None # Compute the weight at each sample location weights = ray_samples.get_weights(densities) if torch.is_tensor(weights): assert not torch.is_tensor(densities), "Cannot use both densities and weights" + assert weights is not None starts = ray_samples.spacing_starts ends = ray_samples.spacing_ends @@ -196,7 +198,7 @@ def orientation_loss( w = weights n = normals v = viewdirs * -1 - n_dot_v = (n * v[..., None, :]).sum(axis=-1) + n_dot_v = (n * v[..., None, :]).sum(dim=-1) return (w[..., 0] * torch.fmin(torch.zeros_like(n_dot_v), n_dot_v) ** 2).sum(dim=-1) @@ -379,7 +381,7 @@ class GradientLoss(nn.Module): reduction_type: either "batch" or "image" """ super().__init__() - self.reduction_type = reduction_type + self.reduction_type: Literal["image", "batch"] = reduction_type self.__scales = scales def forward( @@ -408,7 +410,7 @@ class GradientLoss(nn.Module): ) total += grad_loss - return total + return cast(Tensor, total) def gradient_loss( self, @@ -532,9 +534,12 @@ class _GradientScaler(torch.autograd.Function): # typing: ignore, pylint: disab return output_grad * scaling, grad_scaling +K = TypeVar("K") + + def scale_gradients_by_distance_squared( - field_outputs: Dict[str, torch.Tensor], ray_samples: RaySamples -) -> Dict[str, torch.Tensor]: + field_outputs: Dict[K, torch.Tensor], ray_samples: RaySamples +) -> Dict[K, torch.Tensor]: """ Scale gradients by the ray distance to the pixel as suggested in `Radiance Field Gradient Scaling for Unbiased Near-Camera Training` paper diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 12cc45a57..88440f4f6 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -39,6 +39,7 @@ from nerfstudio.cameras.rays import RaySamples from nerfstudio.utils import colors from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize +BackgroundColor = Union[Literal["random", "last_sample", "black", "white"], Float[Tensor, "3"]] BACKGROUND_COLOR_OVERRIDE: Optional[Float[Tensor, "3"]] = None @@ -61,18 +62,16 @@ class RGBRenderer(nn.Module): background_color: Background color as RGB. Uses random colors if None. """ - def __init__( - self, background_color: Union[Literal["random", "last_sample"], Float[Tensor, "3"]] = "random" - ) -> None: + def __init__(self, background_color: BackgroundColor = "random") -> None: super().__init__() - self.background_color = background_color + self.background_color: BackgroundColor = background_color @classmethod def combine_rgb( cls, rgb: Float[Tensor, "*bs num_samples 3"], weights: Float[Tensor, "*bs num_samples 1"], - background_color: Union[Literal["random", "white", "black", "last_sample"], Float[Tensor, "3"]] = "random", + background_color: BackgroundColor = "random", ray_indices: Optional[Int[Tensor, "num_samples"]] = None, num_rays: Optional[int] = None, ) -> Float[Tensor, "*bs 3"]: @@ -155,11 +154,11 @@ class SHRenderer(nn.Module): def __init__( self, - background_color: Union[Literal["random", "last_sample"], Float[Tensor, "3"]] = "random", + background_color: BackgroundColor = "random", activation: Optional[nn.Module] = nn.Sigmoid(), ) -> None: super().__init__() - self.background_color = background_color + self.background_color: BackgroundColor = background_color self.activation = activation def forward( diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index 78eb96d3d..71d795945 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -74,7 +74,7 @@ class Model(nn.Module): super().__init__() self.config = config self.scene_box = scene_box - self.render_aabb = None # the box that we want to render - should be a subset of scene_box + self.render_aabb: Optional[SceneBox] = None # the box that we want to render - should be a subset of scene_box self.num_train_data = num_train_data self.kwargs = kwargs self.collider = None diff --git a/nerfstudio/models/base_surface_model.py b/nerfstudio/models/base_surface_model.py index 9c68cfaf4..6076da403 100644 --- a/nerfstudio/models/base_surface_model.py +++ b/nerfstudio/models/base_surface_model.py @@ -20,7 +20,7 @@ from __future__ import annotations from abc import abstractmethod from dataclasses import dataclass, field -from typing import Dict, List, Literal, Tuple, Type +from typing import Any, Dict, List, Literal, Tuple, Type import torch import torch.nn.functional as F @@ -173,14 +173,15 @@ class SurfaceModel(Model): def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} param_groups["fields"] = list(self.field.parameters()) - if self.config.background_model != "none": - param_groups["field_background"] = list(self.field_background.parameters()) - else: - param_groups["field_background"] = list(self.field_background) + param_groups["field_background"] = ( + [self.field_background] + if isinstance(self.field_background, Parameter) + else list(self.field_background.parameters()) + ) return param_groups @abstractmethod - def sample_and_forward_field(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]: + def sample_and_forward_field(self, ray_bundle: RayBundle) -> Dict[str, Any]: """Takes in a Ray Bundle and returns a dictionary of samples and field output. Args: @@ -221,10 +222,12 @@ class SurfaceModel(Model): if self.config.background_model != "none": # sample inversely from far to 1000 and points and forward the bg model ray_bundle.nears = ray_bundle.fars + assert ray_bundle.fars is not None ray_bundle.fars = torch.ones_like(ray_bundle.fars) * self.config.far_plane_bg ray_samples_bg = self.sampler_bg(ray_bundle) # use the same background model for both density field and occupancy field + assert not isinstance(self.field_background, Parameter) field_outputs_bg = self.field_background(ray_samples_bg) weights_bg = ray_samples_bg.get_weights(field_outputs_bg[FieldHeadNames.DENSITY]) diff --git a/nerfstudio/models/mipnerf.py b/nerfstudio/models/mipnerf.py index fadf99ea7..776764ac1 100644 --- a/nerfstudio/models/mipnerf.py +++ b/nerfstudio/models/mipnerf.py @@ -36,7 +36,8 @@ from nerfstudio.model_components.renderers import ( DepthRenderer, RGBRenderer, ) -from nerfstudio.models.base_model import Model, ModelConfig +from nerfstudio.models.base_model import Model +from nerfstudio.models.vanilla_nerf import VanillaModelConfig from nerfstudio.utils import colormaps, colors, misc @@ -47,12 +48,15 @@ class MipNerfModel(Model): config: MipNerf configuration to instantiate model """ + config: VanillaModelConfig + def __init__( self, - config: ModelConfig, + config: VanillaModelConfig, **kwargs, ) -> None: self.field = None + assert config.collider_params is not None, "MipNeRF model requires bounding box collider parameters." super().__init__(config=config, **kwargs) def populate_modules(self): @@ -151,6 +155,8 @@ class MipNerfModel(Model): rgb_fine = outputs["rgb_fine"] acc_coarse = colormaps.apply_colormap(outputs["accumulation_coarse"]) acc_fine = colormaps.apply_colormap(outputs["accumulation_fine"]) + + assert self.config.collider_params is not None depth_coarse = colormaps.apply_depth_colormap( outputs["depth_coarse"], accumulation=outputs["accumulation_coarse"], @@ -180,6 +186,7 @@ class MipNerfModel(Model): fine_ssim = self.ssim(image, rgb_fine) fine_lpips = self.lpips(image, rgb_fine) + assert isinstance(fine_ssim, torch.Tensor) metrics_dict = { "psnr": float(fine_psnr.item()), "coarse_psnr": float(coarse_psnr.item()), diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index 233234e89..6d46c86c3 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -271,7 +271,7 @@ class NerfactoModel(Model): def get_outputs(self, ray_bundle: RayBundle): ray_samples: RaySamples ray_samples, weights_list, ray_samples_list = self.proposal_sampler(ray_bundle, density_fns=self.density_fns) - field_outputs = self.field(ray_samples, compute_normals=self.config.predict_normals) + field_outputs = self.field.forward(ray_samples, compute_normals=self.config.predict_normals) if self.config.use_gradient_scaling: field_outputs = scale_gradients_by_distance_squared(field_outputs, ray_samples) diff --git a/nerfstudio/models/vanilla_nerf.py b/nerfstudio/models/vanilla_nerf.py index 44543eb8e..e600ac2ad 100644 --- a/nerfstudio/models/vanilla_nerf.py +++ b/nerfstudio/models/vanilla_nerf.py @@ -67,6 +67,8 @@ class NeRFModel(Model): config: Basic NeRF configuration to instantiate model """ + config: VanillaModelConfig + def __init__( self, config: VanillaModelConfig, @@ -227,6 +229,7 @@ class NeRFModel(Model): fine_psnr = self.psnr(image, rgb_fine) fine_ssim = self.ssim(image, rgb_fine) fine_lpips = self.lpips(image, rgb_fine) + assert isinstance(fine_ssim, torch.Tensor) metrics_dict = { "psnr": float(fine_psnr.item()), diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 4596d57ff..b51d31370 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -186,11 +186,13 @@ class Pipeline(nn.Module): step: training step of the loaded checkpoint """ + @abstractmethod def get_training_callbacks( self, training_callback_attributes: TrainingCallbackAttributes ) -> List[TrainingCallback]: """Returns the training callbacks from both the Dataloader and the Model.""" + @abstractmethod def get_param_groups(self) -> Dict[str, List[Parameter]]: """Get the param groups for the pipeline. diff --git a/nerfstudio/pipelines/dynamic_batch.py b/nerfstudio/pipelines/dynamic_batch.py index c7c4f702a..57e7afa72 100644 --- a/nerfstudio/pipelines/dynamic_batch.py +++ b/nerfstudio/pipelines/dynamic_batch.py @@ -84,11 +84,12 @@ class DynamicBatchPipeline(VanillaPipeline): "'num_samples_per_batch' is not in metrics_dict." "Please return 'num_samples_per_batch' in the models get_metrics_dict function to use this method." ) - self._update_dynamic_num_rays_per_batch(metrics_dict["num_samples_per_batch"]) + self._update_dynamic_num_rays_per_batch(int(metrics_dict["num_samples_per_batch"])) self._update_pixel_samplers() # add the number of rays assert "num_rays_per_batch" not in metrics_dict + assert self.datamanager.train_pixel_sampler is not None metrics_dict["num_rays_per_batch"] = torch.tensor(self.datamanager.train_pixel_sampler.num_rays_per_batch) return model_outputs, loss_dict, metrics_dict @@ -98,6 +99,7 @@ class DynamicBatchPipeline(VanillaPipeline): # add the number of rays assert "num_rays_per_batch" not in metrics_dict + assert self.datamanager.eval_pixel_sampler is not None metrics_dict["num_rays_per_batch"] = torch.tensor(self.datamanager.eval_pixel_sampler.num_rays_per_batch) return model_outputs, loss_dict, metrics_dict diff --git a/nerfstudio/process_data/metashape_utils.py b/nerfstudio/process_data/metashape_utils.py index 470cacb80..c1c2d5093 100644 --- a/nerfstudio/process_data/metashape_utils.py +++ b/nerfstudio/process_data/metashape_utils.py @@ -113,8 +113,8 @@ def metashape_to_json( # pylint: disable=too-many-statements sensor_dict[sensor.get("id")] = s components = chunk.find("components") + component_dict = {} if components is not None: - component_dict = {} for component in components: transform = component.find("transform") if transform is not None: @@ -122,16 +122,19 @@ def metashape_to_json( # pylint: disable=too-many-statements if rotation is None: r = np.eye(3) else: + assert isinstance(rotation.text, str) r = np.array([float(x) for x in rotation.text.split()]).reshape((3, 3)) translation = transform.find("translation") if translation is None: t = np.zeros(3) else: + assert isinstance(translation.text, str) t = np.array([float(x) for x in translation.text.split()]) scale = transform.find("scale") if scale is None: s = 1.0 else: + assert isinstance(scale.text, str) s = float(scale.text) m = np.eye(4) @@ -146,6 +149,7 @@ def metashape_to_json( # pylint: disable=too-many-statements for camera in cameras: frame = {} camera_label = camera.get("label") + assert isinstance(camera_label, str) if camera_label not in image_filename_map: # Labels sometimes have a file extension. Try without the extension. # (maybe it's just a '.' in the image name) @@ -169,11 +173,12 @@ def metashape_to_json( # pylint: disable=too-many-statements CONSOLE.print(f"Missing transforms data for {camera.get('label')}, Skipping") num_skipped += 1 continue - transform = np.array([float(x) for x in camera.find("transform").text.split()]).reshape((4, 4)) - if components is not None: - component_id = camera.get("component_id") - if component_id in component_dict: - transform = component_dict[component_id] @ transform + transform = np.array([float(x) for x in camera.find("transform").text.split()]).reshape((4, 4)) # type: ignore + + component_id = camera.get("component_id") + if component_id in component_dict: + transform = component_dict[component_id] @ transform + transform = transform[[2, 0, 1, 3], :] transform[:, 1:3] *= -1 frame["transform_matrix"] = transform.tolist() diff --git a/nerfstudio/process_data/process_data_utils.py b/nerfstudio/process_data/process_data_utils.py index ffa379a9d..15c414dcc 100644 --- a/nerfstudio/process_data/process_data_utils.py +++ b/nerfstudio/process_data/process_data_utils.py @@ -20,7 +20,7 @@ import shutil import sys from enum import Enum from pathlib import Path -from typing import List, Literal, Optional, OrderedDict, Tuple +from typing import List, Literal, Optional, OrderedDict, Tuple, Union import cv2 import numpy as np @@ -381,7 +381,23 @@ def find_tool_feature_matcher_combination( matcher_type: Literal[ "any", "NN", "superglue", "superglue-fast", "NN-superpoint", "NN-ratio", "NN-mutual", "adalam" ], -): +) -> Union[ + Tuple[None, None, None], + Tuple[ + Literal["colmap", "hloc"], + Literal[ + "sift", + "superpoint_aachen", + "superpoint_max", + "superpoint_inloc", + "r2d2", + "d2net-ss", + "sosnet", + "disk", + ], + Literal["superglue", "superglue-fast", "NN-superpoint", "NN-ratio", "NN-mutual", "adalam"], + ], +]: """Find a valid combination of sfm tool, feature type, and matcher type. Basically, replace the default parameters 'any' by usable value @@ -436,7 +452,7 @@ def generate_circle_mask(height: int, width: int, percent_radius) -> Optional[np mask = np.zeros((height, width), dtype=np.uint8) center = (width // 2, height // 2) radius = int(percent_radius * np.sqrt(width**2 + height**2) / 2.0) - cv2.circle(mask, center, radius, 1, -1) + cv2.circle(mask, center, radius, 1, -1) # type: ignore return mask @@ -507,7 +523,7 @@ def save_mask( The path to the mask file or None if no mask is needed. """ image_path = next(image_dir.glob("frame_*")) - image = cv2.imread(str(image_path)) + image = cv2.imread(str(image_path)) # type: ignore height, width = image.shape[:2] mask = generate_mask(height, width, crop_factor, percent_radius) if mask is None: @@ -515,13 +531,17 @@ def save_mask( mask *= 255 mask_path = image_dir.parent / "masks" mask_path.mkdir(exist_ok=True) - cv2.imwrite(str(mask_path / "mask.png"), mask) + cv2.imwrite(str(mask_path / "mask.png"), mask) # type: ignore downscale_factors = [2**i for i in range(num_downscales + 1)[1:]] for downscale in downscale_factors: mask_path_i = image_dir.parent / f"masks_{downscale}" mask_path_i.mkdir(exist_ok=True) mask_path_i = mask_path_i / "mask.png" - mask_i = cv2.resize(mask, (width // downscale, height // downscale), interpolation=cv2.INTER_NEAREST) - cv2.imwrite(str(mask_path_i), mask_i) + mask_i = cv2.resize( # type: ignore + mask, + (width // downscale, height // downscale), + interpolation=cv2.INTER_NEAREST, # type: ignore + ) + cv2.imwrite(str(mask_path_i), mask_i) # type: ignore CONSOLE.log(":tada: Generated and saved masks.") return mask_path / "mask.png" diff --git a/nerfstudio/scripts/blender/nerfstudio_blender.py b/nerfstudio/scripts/blender/nerfstudio_blender.py index a936edcd5..0da0fc2da 100644 --- a/nerfstudio/scripts/blender/nerfstudio_blender.py +++ b/nerfstudio/scripts/blender/nerfstudio_blender.py @@ -1,3 +1,5 @@ +# type: ignore + # Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/nerfstudio/scripts/datasets/process_nuscenes_masks.py b/nerfstudio/scripts/datasets/process_nuscenes_masks.py index f88588595..77efc3432 100644 --- a/nerfstudio/scripts/datasets/process_nuscenes_masks.py +++ b/nerfstudio/scripts/datasets/process_nuscenes_masks.py @@ -53,7 +53,11 @@ class ProcessNuScenesMasks: def main(self) -> None: """Generate NuScenes dynamic object masks.""" - nusc = NuScenesDatabase(version=self.version, dataroot=self.data_dir, verbose=self.verbose) + nusc = NuScenesDatabase( + version=self.version, + dataroot=str(self.data_dir.absolute()), + verbose=self.verbose, + ) cameras = ["CAM_" + camera for camera in self.cameras] for camera in cameras: @@ -105,18 +109,18 @@ class ProcessNuScenesMasks: corners_3d = box.corners() corners = view_points(corners_3d, intrinsics, normalize=True)[:2, :] corners = np.round(corners).astype(int).T - cv2.fillPoly(mask, [corners[[0, 1, 2, 3]]], 0) # front - cv2.fillPoly(mask, [corners[[4, 5, 6, 7]]], 0) # back - cv2.fillPoly(mask, [corners[[0, 1, 5, 4]]], 0) # top - cv2.fillPoly(mask, [corners[[2, 3, 7, 6]]], 0) # bottom - cv2.fillPoly(mask, [corners[[0, 3, 7, 4]]], 0) # left - cv2.fillPoly(mask, [corners[[1, 2, 6, 5]]], 0) # right + cv2.fillPoly(mask, [corners[[0, 1, 2, 3]]], 0) # front # type: ignore + cv2.fillPoly(mask, [corners[[4, 5, 6, 7]]], 0) # back # type: ignore + cv2.fillPoly(mask, [corners[[0, 1, 5, 4]]], 0) # top # type: ignore + cv2.fillPoly(mask, [corners[[2, 3, 7, 6]]], 0) # bottom # type: ignore + cv2.fillPoly(mask, [corners[[0, 3, 7, 4]]], 0) # left # type: ignore + cv2.fillPoly(mask, [corners[[1, 2, 6, 5]]], 0) # right # type: ignore maskname = os.path.split(camera_data["filename"])[1].replace("jpg", "png") - cv2.imwrite(str(self.output_dir / "masks" / camera / maskname), mask * 255) + cv2.imwrite(str(self.output_dir / "masks" / camera / maskname), mask * 255) # type: ignore if self.verbose: - img = cv2.imread(str(self.data_dir / camera_data["filename"])) + img = cv2.imread(str(self.data_dir / camera_data["filename"])) # type: ignore mask = ~mask.astype(bool) img[mask, :] -= np.minimum(img[mask, :], 100) viz.append(img) @@ -124,16 +128,16 @@ class ProcessNuScenesMasks: if self.verbose: if len(viz) == 6: viz = np.vstack((np.hstack(viz[:3]), np.hstack(viz[3:]))) - viz = cv2.resize(viz, (int(1600 * 3 / 3), int(900 * 2 / 3))) + viz = cv2.resize(viz, (int(1600 * 3 / 3), int(900 * 2 / 3))) # type: ignore elif len(viz) == 3: viz = np.hstack(viz[:3]) - viz = cv2.resize(viz, (int(1600 * 3 / 3), int(900 / 3))) + viz = cv2.resize(viz, (int(1600 * 3 / 3), int(900 / 3))) # type: ignore elif len(viz) == 1: viz = viz[0] else: raise ValueError("Only support 1 or 3 or 6 cameras for viz") - cv2.imshow("", viz) - cv2.waitKey(1) + cv2.imshow("", viz) # type: ignore + cv2.waitKey(1) # type: ignore def entrypoint(): diff --git a/nerfstudio/scripts/downloads/download_data.py b/nerfstudio/scripts/downloads/download_data.py index 8c9a3216e..e725bbca6 100644 --- a/nerfstudio/scripts/downloads/download_data.py +++ b/nerfstudio/scripts/downloads/download_data.py @@ -21,7 +21,7 @@ import tarfile import zipfile from dataclasses import dataclass from pathlib import Path -from typing import Union +from typing import TYPE_CHECKING, Union import gdown import tyro @@ -135,7 +135,10 @@ nerfstudio_file_ids = { "nerfstudio-dataset": nerfstudio_dataset, } -NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_ids.keys()) +if TYPE_CHECKING: + NerfstudioCaptureName = str +else: + NerfstudioCaptureName = tyro.extras.literal_type_from_choices(nerfstudio_file_ids.keys()) def download_capture_name(save_dir: Path, dataset_name: str, capture_name: str, capture_name_to_file_id: dict): @@ -206,7 +209,10 @@ record3d_file_ids = { "bear": grab_file_id("https://drive.google.com/file/d/1WRZohWMRj0nNlYFIEBwkddDoGPvLTzkR/view?usp=sharing") } -Record3dCaptureName = tyro.extras.literal_type_from_choices(record3d_file_ids.keys()) +if TYPE_CHECKING: + Record3dCaptureName = str +else: + Record3dCaptureName = tyro.extras.literal_type_from_choices(record3d_file_ids.keys()) @dataclass @@ -257,7 +263,10 @@ phototourism_downloads = { "all": None, } -PhototourismCaptureName = tyro.extras.literal_type_from_choices(phototourism_downloads.keys()) +if TYPE_CHECKING: + PhototourismCaptureName = str +else: + PhototourismCaptureName = tyro.extras.literal_type_from_choices(phototourism_downloads.keys()) @dataclass @@ -315,7 +324,10 @@ sdfstudio_downloads = { "all": None, } -SDFstudioCaptureName = tyro.extras.literal_type_from_choices(sdfstudio_downloads.keys()) +if TYPE_CHECKING: + SDFstudioCaptureName = str +else: + SDFstudioCaptureName = tyro.extras.literal_type_from_choices(sdfstudio_downloads.keys()) @dataclass @@ -385,7 +397,10 @@ nerfosr_downloads = { } -NeRFOSRCaptureName = tyro.extras.literal_type_from_choices(nerfosr_downloads.keys()) +if TYPE_CHECKING: + NeRFOSRCaptureName = str +else: + NeRFOSRCaptureName = tyro.extras.literal_type_from_choices(nerfosr_downloads.keys()) @dataclass diff --git a/nerfstudio/scripts/exporter.py b/nerfstudio/scripts/exporter.py index 09e0de716..8a4101115 100644 --- a/nerfstudio/scripts/exporter.py +++ b/nerfstudio/scripts/exporter.py @@ -25,7 +25,7 @@ import os import sys from dataclasses import dataclass, field from pathlib import Path -from typing import List, Optional, Tuple, Union +from typing import List, Optional, Tuple, Union, cast import numpy as np import open3d as o3d @@ -34,6 +34,7 @@ import tyro from typing_extensions import Annotated, Literal from nerfstudio.cameras.rays import RayBundle +from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManager from nerfstudio.exporter import texture_utils, tsdf_utils from nerfstudio.exporter.exporter_utils import ( collect_camera_poses, @@ -43,6 +44,7 @@ from nerfstudio.exporter.exporter_utils import ( from nerfstudio.exporter.marching_cubes import ( generate_mesh_with_multires_marching_cubes, ) +from nerfstudio.fields.sdf_field import SDFField from nerfstudio.pipelines.base_pipeline import Pipeline, VanillaPipeline from nerfstudio.utils.eval_utils import eval_setup from nerfstudio.utils.rich_utils import CONSOLE @@ -92,6 +94,8 @@ class ExportPointCloud(Exporter): _, pipeline, _, _ = eval_setup(self.load_config) # Increase the batchsize to speed up the evaluation. + assert isinstance(pipeline.datamanager, VanillaDataManager) + assert pipeline.datamanager.train_pixel_sampler is not None pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch pcd = generate_point_cloud( @@ -114,7 +118,7 @@ class ExportPointCloud(Exporter): tpcd = o3d.t.geometry.PointCloud.from_legacy(pcd) # The legacy PLY writer converts colors to UInt8, # let us do the same to save space. - tpcd.point.colors = (tpcd.point.colors * 255).to(o3d.core.Dtype.UInt8) + tpcd.point.colors = (tpcd.point.colors * 255).to(o3d.core.Dtype.UInt8) # type: ignore o3d.t.io.write_point_cloud(str(self.output_dir / "point_cloud.ply"), tpcd) print("\033[A\033[A") CONSOLE.print("[bold green]:white_check_mark: Saving Point Cloud") @@ -269,6 +273,8 @@ class ExportPoissonMesh(Exporter): self.validate_pipeline(pipeline) # Increase the batchsize to speed up the evaluation. + assert isinstance(pipeline.datamanager, VanillaDataManager) + assert pipeline.datamanager.train_pixel_sampler is not None pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch # Whether the normals should be estimated based on the point cloud. @@ -369,7 +375,9 @@ class ExportMarchingCubesMesh(Exporter): # Extract mesh using marching cubes for sdf at a multi-scale resolution. multi_res_mesh = generate_mesh_with_multires_marching_cubes( - geometry_callable_field=lambda x: pipeline.model.field.forward_geonetwork(x)[:, 0].contiguous(), + geometry_callable_field=lambda x: cast(SDFField, pipeline.model.field) + .forward_geonetwork(x)[:, 0] + .contiguous(), resolution=self.resolution, bounding_box_min=self.bounding_box_min, bounding_box_max=self.bounding_box_max, diff --git a/nerfstudio/scripts/render.py b/nerfstudio/scripts/render.py index d97d6b6c7..5ba8371f3 100644 --- a/nerfstudio/scripts/render.py +++ b/nerfstudio/scripts/render.py @@ -366,8 +366,10 @@ class RenderInterpolated(BaseRender): install_checks.check_ffmpeg_installed() if self.pose_source == "eval": + assert pipeline.datamanager.eval_dataset is not None cameras = pipeline.datamanager.eval_dataset.cameras else: + assert pipeline.datamanager.train_dataset is not None cameras = pipeline.datamanager.train_dataset.cameras seconds = self.interpolation_steps * len(cameras) / self.frame_rate diff --git a/nerfstudio/utils/colormaps.py b/nerfstudio/utils/colormaps.py index 27b7e4e21..0e09c0bfd 100644 --- a/nerfstudio/utils/colormaps.py +++ b/nerfstudio/utils/colormaps.py @@ -24,7 +24,7 @@ from torch import Tensor from nerfstudio.utils import colors -Colormaps = Literal["default", "turbo", "viridis", "magma", "inferno", "cividis"] +Colormaps = Literal["default", "turbo", "viridis", "magma", "inferno", "cividis", "pca"] @dataclass(frozen=True) @@ -103,15 +103,13 @@ def apply_float_colormap(image: Float[Tensor, "*bs 1"], colormap: Colormaps = "v if colormap == "default": colormap = "turbo" - colormap = matplotlib.colormaps[colormap] - colormap = torch.tensor(colormap.colors).to(image.device) # type: ignore image = torch.nan_to_num(image, 0) image_long = (image * 255).long() image_long_min = torch.min(image_long) image_long_max = torch.max(image_long) assert image_long_min >= 0, f"the min value is {image_long_min}" assert image_long_max <= 255, f"the max value is {image_long_max}" - return colormap[image_long[..., 0]] + return torch.tensor(matplotlib.colormaps[colormap].colors, device=image.device)[image_long[..., 0]] def apply_depth_colormap( diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py index 9497756aa..4d531ba45 100644 --- a/nerfstudio/utils/eval_utils.py +++ b/nerfstudio/utils/eval_utils.py @@ -25,6 +25,7 @@ from typing import Literal, Optional, Tuple import torch import yaml +from nerfstudio.data.datamanagers.base_datamanager import VanillaDataManagerConfig from nerfstudio.engine.trainer import TrainerConfig from nerfstudio.pipelines.base_pipeline import Pipeline from nerfstudio.utils.rich_utils import CONSOLE @@ -92,7 +93,8 @@ def eval_setup( # load checkpoints from wherever they were saved # TODO: expose the ability to choose an arbitrary checkpoint config.load_dir = config.get_checkpoint_dir() - config.pipeline.datamanager.eval_image_indices = None + if isinstance(config.pipeline.datamanager, VanillaDataManagerConfig): + config.pipeline.datamanager.eval_image_indices = None # setup pipeline (which includes the DataManager) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index 7f46f31e9..69a9de4d0 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -258,7 +258,7 @@ def masked_reduction( input_tensor: Float[Tensor, "1 32 mult"], mask: Bool[Tensor, "1 32 mult"], reduction_type: Literal["image", "batch"], -): +) -> Tensor: """ Whether to consolidate the input_tensor across the batch or across the image Args: @@ -271,7 +271,9 @@ def masked_reduction( if reduction_type == "batch": # avoid division by 0 (if sum(M) = sum(sum(mask)) = 0: sum(image_loss) = 0) divisor = torch.sum(mask) - input_tensor = 0 if divisor == 0 else torch.sum(input_tensor) / divisor + if divisor == 0: + return torch.tensor(0, device=input_tensor.device) + input_tensor = torch.sum(input_tensor) / divisor elif reduction_type == "image": # avoid division by 0 (if M = sum(mask) = 0: image_loss = 0) valid = mask.nonzero() diff --git a/nerfstudio/utils/plotly_utils.py b/nerfstudio/utils/plotly_utils.py index aa08a6008..8eff73a25 100644 --- a/nerfstudio/utils/plotly_utils.py +++ b/nerfstudio/utils/plotly_utils.py @@ -82,7 +82,7 @@ def get_line_segments_from_lines( marker_color = colors[idx] thiscolor = colors[idx] data.append( - go.Scatter3d( + go.Scatter3d( # type: ignore x=line[:, 0], y=line[:, 1], z=line[:, 2], @@ -100,7 +100,10 @@ def get_line_segments_from_lines( return data -def vis_dataset(camera_origins: Float[Tensor, "num_cameras 3"], ray_bundle: RayBundle) -> go.FigureWidget: +def vis_dataset( + camera_origins: Float[Tensor, "num_cameras 3"], + ray_bundle: RayBundle, +) -> go.FigureWidget: # type: ignore """Visualize a dataset with plotly using our cameras and generated rays. Args: @@ -117,7 +120,7 @@ def vis_dataset(camera_origins: Float[Tensor, "num_cameras 3"], ray_bundle: RayB data = [] data += [ - go.Scatter3d( + go.Scatter3d( # type: ignore x=camera_origins[::skip, 0], y=camera_origins[::skip, 1], z=camera_origins[::skip, 2], @@ -144,7 +147,7 @@ def vis_dataset(camera_origins: Float[Tensor, "num_cameras 3"], ray_bundle: RayB camera=dict(up=dict(x=0, y=0, z=1), center=dict(x=0, y=0, z=0), eye=dict(x=1.25, y=1.25, z=1.25)), ), ) - fig = go.Figure(data=data, layout=layout) + fig = go.Figure(data=data, layout=layout) # type: ignore return fig @@ -166,7 +169,11 @@ def get_random_color(colormap: Optional[List[str]] = None, idx: Optional[int] = def get_sphere( - radius: float, center: Float[Tensor, "3"] = None, color: str = "black", opacity: float = 1.0, resolution: int = 32 + radius: float, + center: Optional[Float[Tensor, "3"]] = None, + color: str = "black", + opacity: float = 1.0, + resolution: int = 32, ) -> go.Mesh3d: # type: ignore """Returns a sphere object for plotting with plotly. @@ -193,7 +200,7 @@ def get_sphere( if center is not None: pts += center - return go.Mesh3d( + return go.Mesh3d( # type: ignore { "x": pts[:, :, 0].flatten(), "y": pts[:, :, 1].flatten(), @@ -207,7 +214,7 @@ def get_sphere( def get_cube( side_length: Union[float, torch.Tensor], - center: Float[Tensor, "3"] = None, + center: Optional[Float[Tensor, "3"]] = None, color: str = "black", opacity: float = 1.0, ) -> go.Mesh3d: # type: ignore @@ -241,7 +248,7 @@ def get_cube( pts[1] = np.add(pts[1], center[1]) pts[2] = np.add(pts[2], center[2]) - return go.Mesh3d( + return go.Mesh3d( # type: ignore { "x": pts[0], "y": pts[1], @@ -299,7 +306,7 @@ def get_gaussian_ellipsiod( pts += mean - return go.Mesh3d( + return go.Mesh3d( # type: ignore { "x": pts[:, :, 0].flatten(), "y": pts[:, :, 1].flatten(), @@ -328,7 +335,7 @@ def get_gaussian_ellipsoids_list( """ data = [] - vis_means = go.Scatter3d( + vis_means = go.Scatter3d( # type: ignore x=gaussians.mean[:, 0], y=gaussians.mean[:, 1], z=gaussians.mean[:, 2], @@ -394,7 +401,7 @@ def get_frustum_mesh( pts = torch.einsum("kj,ij->ki", pts, rotation) pts += frustum.origins - return go.Mesh3d( + return go.Mesh3d( # type: ignore x=pts[..., 0], y=pts[..., 1], z=pts[..., 2], @@ -448,7 +455,7 @@ def get_frustum_points( frustum = frustum.flatten() pts = frustum.get_positions() - return go.Scatter3d( + return go.Scatter3d( # type: ignore x=pts[..., 0], y=pts[..., 1], z=pts[..., 2], @@ -483,7 +490,7 @@ def get_ray_bundle_lines( lines = torch.empty((origins.shape[0] * 2, 3)) lines[0::2] = origins lines[1::2] = origins + directions * length - return go.Scatter3d( + return go.Scatter3d( # type: ignore x=lines[..., 0], y=lines[..., 1], z=lines[..., 2], @@ -522,8 +529,8 @@ def vis_camera_rays(cameras: Cameras) -> go.Figure: # type: ignore colors[0::2] = coords colors[1::2] = coords - fig = go.Figure( - data=go.Scatter3d( + fig = go.Figure( # type: ignore + data=go.Scatter3d( # type: ignore x=lines[:, 0], y=lines[:, 2], z=lines[:, 1], diff --git a/nerfstudio/utils/profiler.py b/nerfstudio/utils/profiler.py index 088b77487..a119f923c 100644 --- a/nerfstudio/utils/profiler.py +++ b/nerfstudio/utils/profiler.py @@ -24,7 +24,18 @@ import typing from collections import deque from contextlib import ContextDecorator, contextmanager from pathlib import Path -from typing import Callable, Dict, List, Optional, Tuple, Union +from typing import ( + Any, + Callable, + ContextManager, + Dict, + List, + Optional, + Tuple, + TypeVar, + Union, + overload, +) from torch.profiler import ProfilerActivity, profile, record_function @@ -41,7 +52,32 @@ PROFILER = [] PYTORCH_PROFILER = None -class time_function(ContextDecorator): # pylint: disable=invalid-name +CallableT = TypeVar("CallableT", bound=Callable) + + +@overload +def time_function(name_or_func: CallableT) -> CallableT: + ... + + +@overload +def time_function(name_or_func: str) -> ContextManager[Any]: + ... + + +def time_function(name_or_func: Union[CallableT, str]) -> Union[CallableT, ContextManager[Any]]: + """Profile a function or block of code. Can be used either to create a context or to wrap a function. + + Args: + name_or_func: Either the name of a context or function to profile. + + Returns: + A wrapped function or context to use in a `with` statement. + """ + return _TimeFunction(name_or_func) + + +class _TimeFunction(ContextDecorator): # pylint: disable=invalid-name """Decorator/Context manager: time a function call or a block of code""" def __init__(self, name: Union[str, Callable]): diff --git a/nerfstudio/utils/rich_utils.py b/nerfstudio/utils/rich_utils.py index 95ee43cb7..cb9fa23fe 100644 --- a/nerfstudio/utils/rich_utils.py +++ b/nerfstudio/utils/rich_utils.py @@ -22,6 +22,7 @@ from rich.progress import ( BarColumn, Progress, ProgressColumn, + Task, TaskProgressColumn, TextColumn, TimeRemainingColumn, @@ -38,7 +39,7 @@ class ItersPerSecColumn(ProgressColumn): super().__init__() self.suffix = suffix - def render(self, task: "Task") -> Text: + def render(self, task: Task) -> Text: """Show data transfer speed.""" speed = task.finished_speed or task.speed if speed is None: diff --git a/nerfstudio/utils/tensor_dataclass.py b/nerfstudio/utils/tensor_dataclass.py index 7eca91610..cbf7d5533 100644 --- a/nerfstudio/utils/tensor_dataclass.py +++ b/nerfstudio/utils/tensor_dataclass.py @@ -76,23 +76,23 @@ class TensorDataclass: isinstance(v, int) and v > 1 ), f"Custom dimensions must be an integer greater than 1, since 1 is the default, received {k}: {v}" - if not dataclasses.is_dataclass(self): + # Shim to prevent pyright from narrowing `self` to DataclassInstance. + self_dc = self + if not dataclasses.is_dataclass(self_dc): raise TypeError("TensorDataclass must be a dataclass") - batch_shapes = self._get_dict_batch_shapes( - {f.name: self.__getattribute__(f.name) for f in dataclasses.fields(self)} - ) + batch_shapes = self._get_dict_batch_shapes({f.name: getattr(self, f.name) for f in dataclasses.fields(self_dc)}) if len(batch_shapes) == 0: raise ValueError("TensorDataclass must have at least one tensor") batch_shape = torch.broadcast_shapes(*batch_shapes) broadcasted_fields = self._broadcast_dict_fields( - {f.name: self.__getattribute__(f.name) for f in dataclasses.fields(self)}, batch_shape + {f.name: getattr(self, f.name) for f in dataclasses.fields(self_dc)}, batch_shape ) for f, v in broadcasted_fields.items(): - self.__setattr__(f, v) + object.__setattr__(self, f, v) - self.__setattr__("_shape", batch_shape) + object.__setattr__(self, "_shape", batch_shape) def _get_dict_batch_shapes(self, dict_: Dict) -> List: """Returns batch shapes of all tensors in a dictionary @@ -289,14 +289,17 @@ class TensorDataclass: A new TensorDataclass with the same data but with a new shape. """ + self_dc = self + assert dataclasses.is_dataclass(self_dc) + new_fields = self._apply_fn_to_dict( - {f.name: self.__getattribute__(f.name) for f in dataclasses.fields(self)}, + {f.name: getattr(self, f.name) for f in dataclasses.fields(self_dc)}, fn, dataclass_fn, custom_tensor_dims_fn, ) - return dataclasses.replace(self, **new_fields) + return dataclasses.replace(self_dc, **new_fields) def _apply_fn_to_dict( self, diff --git a/nerfstudio/viewer/server/utils.py b/nerfstudio/viewer/server/utils.py index 100fa1c27..0c9d894ab 100644 --- a/nerfstudio/viewer/server/utils.py +++ b/nerfstudio/viewer/server/utils.py @@ -15,16 +15,17 @@ """Generic utility functions """ -from typing import Callable, List, Optional, Tuple +from typing import List, Optional, Tuple, Union import numpy as np import torch +from typing_extensions import assert_never from nerfstudio.viewer.viser.messages import CameraMessage def get_chunks( - lst: List[float], num_chunks: Optional[int] = None, size_of_chunk: Optional[int] = None + lst: Union[List[float], Tuple[float, ...]], num_chunks: Optional[int] = None, size_of_chunk: Optional[int] = None ) -> List[List[float]]: """Returns list of n elements, containing a sublist. @@ -36,9 +37,12 @@ def get_chunks( if num_chunks: assert not size_of_chunk size = len(lst) // num_chunks - if size_of_chunk: + elif size_of_chunk: assert not num_chunks size = size_of_chunk + else: + assert False, "Either `num_chunks` or `size_of_chunk` must be set" + chunks = [] for i in range(0, len(lst), size): chunks.append(lst[i : i + size]) @@ -61,7 +65,7 @@ def three_js_perspective_camera_focal_length(fov: float, image_height: int): def get_intrinsics_matrix_and_camera_to_world_h( - camera_message: CameraMessage, image_height: int, image_width: Optional[int] = None + camera_message: CameraMessage, image_height: int, image_width: Optional[Union[int, float]] = None ) -> Tuple[torch.Tensor, torch.Tensor]: """Returns the camera intrinsics matrix and the camera to world homogeneous matrix. @@ -76,7 +80,7 @@ def get_intrinsics_matrix_and_camera_to_world_h( image_width = aspect * image_height pp_w = image_width / 2.0 pp_h = image_height / 2.0 - if (camera_message.camera_type == "perspective") | (camera_message.camera_type == "fisheye"): + if camera_message.camera_type in ("perspective", "fisheye"): focal_length = three_js_perspective_camera_focal_length(fov, image_height) intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float() elif camera_message.camera_type == "equirectangular": @@ -89,6 +93,8 @@ def get_intrinsics_matrix_and_camera_to_world_h( intrinsics_matrix = torch.tensor( [[image_height * render_aspect / 2, 0, pp_w], [0, pp_h * 2, pp_h], [0, 0, 1]] ).float() + else: + assert_never(camera_message.camera_type) # extrinsics camera_to_world_h = torch.tensor(get_chunks(camera_message.matrix, size_of_chunk=4)).T.float() @@ -103,23 +109,3 @@ def get_intrinsics_matrix_and_camera_to_world_h( ) return intrinsics_matrix, camera_to_world_h - - -def find_available_port(func: Callable, default_port: int, max_attempts: int = 1000, **kwargs) -> None: - """Finds and attempts to connect to a port - - Args: - func: function used on connecting to port - default_port: the default port - max_attempts: max number of attempts to try connection. Defaults to MAX_ATTEMPTS. - """ - for i in range(max_attempts): - port = default_port + i - try: - return func(port, **kwargs), port - except Exception as e: - print(type(e)) - raise - raise ( - Exception(f"Could not find an available port in the range: [{default_port:d}, {max_attempts + default_port:d})") - ) diff --git a/nerfstudio/viewer/server/viewer_elements.py b/nerfstudio/viewer/server/viewer_elements.py index e377cf07b..be5120aa2 100644 --- a/nerfstudio/viewer/server/viewer_elements.py +++ b/nerfstudio/viewer/server/viewer_elements.py @@ -284,7 +284,7 @@ class ViewerCheckbox(ViewerParameter[bool]): self.gui_handle.set_disabled(self.disabled) -TString = TypeVar("TString", default=str) +TString = TypeVar("TString", default=str, bound=str) class ViewerDropdown(ViewerParameter[TString], Generic[TString]): @@ -317,10 +317,15 @@ class ViewerDropdown(ViewerParameter[TString], Generic[TString]): def _create_gui_handle(self, viser_server: ViserServer) -> None: assert self.gui_handle is None, "gui_handle should be initialized once" - self.gui_handle = viser_server.add_gui_select(self.name, self.options, self.default_value, hint=self.hint) + self.gui_handle = viser_server.add_gui_select( + self.name, + self.options, # type: ignore + self.default_value, # type: ignore + hint=self.hint, + ) self.gui_handle.set_disabled(self.disabled) - def set_options(self, new_options: List[str]) -> None: + def set_options(self, new_options: List[TString]) -> None: """ Sets the options of the dropdown, @@ -329,7 +334,7 @@ class ViewerDropdown(ViewerParameter[TString], Generic[TString]): """ self.options = new_options if self.gui_handle is not None: - self.gui_handle.set_options(new_options) + self.gui_handle.set_options(new_options) # type: ignore class ViewerButtonGroup(ViewerParameter[TString], Generic[TString]): diff --git a/nerfstudio/viewer/server/viewer_state.py b/nerfstudio/viewer/server/viewer_state.py index ea68ec634..7c45d2152 100644 --- a/nerfstudio/viewer/server/viewer_state.py +++ b/nerfstudio/viewer/server/viewer_state.py @@ -176,7 +176,8 @@ class ViewerState: def _crop_params_update(self, _) -> None: """Update crop parameters""" - self.render_statemachine.action(RenderAction("rerender", self.camera_message)) + if self.camera_message is not None: + self.render_statemachine.action(RenderAction("rerender", self.camera_message)) crop_min = torch.tensor(self.control_panel.crop_min, dtype=torch.float32) crop_max = torch.tensor(self.control_panel.crop_max, dtype=torch.float32) scene_box = SceneBox(aabb=torch.stack([crop_min, crop_max], dim=0)) @@ -281,7 +282,7 @@ class ViewerState: # draw indices, roughly evenly spaced return np.linspace(0, total_num - 1, num_display_images, dtype=np.int32).tolist() - def init_scene(self, dataset: InputDataset, train_state=Literal["training", "paused", "completed"]) -> None: + def init_scene(self, dataset: InputDataset, train_state: Literal["training", "paused", "completed"]) -> None: """Draw some images and the scene aabb in the viewer. Args: diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 6b18391a9..9193d4811 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -119,13 +119,13 @@ def update_render_aabb( """ if crop_viewport: - crop_min = torch.tensor(crop_min, dtype=torch.float32) - crop_max = torch.tensor(crop_max, dtype=torch.float32) + crop_min_tensor = torch.tensor(crop_min, dtype=torch.float32) + crop_max_tensor = torch.tensor(crop_max, dtype=torch.float32) if isinstance(model.render_aabb, SceneBox): - model.render_aabb.aabb[0] = crop_min - model.render_aabb.aabb[1] = crop_max + model.render_aabb.aabb[0] = crop_min_tensor + model.render_aabb.aabb[1] = crop_max_tensor else: - model.render_aabb = SceneBox(aabb=torch.stack([crop_min, crop_max], dim=0)) + model.render_aabb = SceneBox(aabb=torch.stack([crop_min_tensor, crop_max_tensor], dim=0)) else: model.render_aabb = None diff --git a/nerfstudio/viewer/viser/message_api.py b/nerfstudio/viewer/viser/message_api.py index 007cc6ead..c0ccce9f3 100644 --- a/nerfstudio/viewer/viser/message_api.py +++ b/nerfstudio/viewer/viser/message_api.py @@ -500,7 +500,12 @@ class MessageApi(abc.ABC): Args: scene_box: The scene box. """ - self._queue(messages.SceneBoxMessage(min=scene_box.aabb[0].tolist(), max=scene_box.aabb[1].tolist())) + self._queue( + messages.SceneBoxMessage( + min=tuple(scene_box.aabb[0].tolist()), + max=tuple(scene_box.aabb[1].tolist()), + ) + ) def add_dataset_image(self, idx: str, json: Dict) -> None: """Add a dataset image to the scene. diff --git a/pyproject.toml b/pyproject.toml index 9e9deb9c4..9b43c5ea3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -79,6 +79,7 @@ dev = [ "pytest==7.1.2", "pytest-xdist==2.5.0", "typeguard==2.13.3", + "sshconf==0.2.5", ] # Documentation related packages @@ -165,7 +166,6 @@ defineConstant = { DEBUG = true } reportMissingImports = true reportMissingTypeStubs = false reportPrivateImportUsage = false -reportUndefinedVariable = false pythonVersion = "3.8" pythonPlatform = "Linux"