From 6c45f72c57e0041769a9b9d2f6e02c34737aece6 Mon Sep 17 00:00:00 2001 From: Matthew Tancik Date: Sat, 30 Jul 2022 15:38:08 -0700 Subject: [PATCH] Typecheck workspace (#231) --- .vscode/settings.json | 7 +-- nerfactory/cameras/rays.py | 3 +- nerfactory/data/structs.py | 4 +- nerfactory/engine/trainer.py | 20 +++++---- .../fields/density_fields/density_grid.py | 11 +++-- nerfactory/fields/instant_ngp_field.py | 7 --- nerfactory/fields/nerf_field.py | 4 +- nerfactory/fields/nerfw_field.py | 8 +++- nerfactory/graphs/base.py | 43 ++++++++++++------- nerfactory/graphs/instant_ngp.py | 10 ++++- nerfactory/graphs/mipnerf.py | 8 +++- nerfactory/graphs/mipnerf_360.py | 10 ++++- nerfactory/graphs/modules/ray_sampler.py | 20 ++++++--- nerfactory/graphs/nerfw.py | 6 +++ nerfactory/graphs/semantic_nerf.py | 6 +-- nerfactory/graphs/vanilla_nerf.py | 21 ++++++--- nerfactory/utils/io.py | 10 +++-- nerfactory/utils/plotly.py | 37 ++++++++-------- nerfactory/utils/visualization.py | 6 +-- 19 files changed, 151 insertions(+), 90 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 4e0857652..6a295ac34 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -11,8 +11,8 @@ "[python]": { "editor.defaultFormatter": "ms-python.python", "editor.codeActionsOnSave": { - "source.organizeImports": true - }, + "source.organizeImports": true + } }, "editor.formatOnSave": true, "python.envFile": "${workspaceFolder}/.env", @@ -94,5 +94,6 @@ "ios": "cpp", "__atomic": "cpp", "__node_handle": "cpp" - } + }, + "python.analysis.typeCheckingMode": "basic" } diff --git a/nerfactory/cameras/rays.py b/nerfactory/cameras/rays.py index b5d221fcf..19d119fcb 100644 --- a/nerfactory/cameras/rays.py +++ b/nerfactory/cameras/rays.py @@ -159,6 +159,7 @@ class RayBundle(TensorDataclass): nears: Distance along ray to start sampling fars: Rays Distance along ray to stop sampling valid_mask: Rays that are valid + num_rays_per_chunk: Number of rays per chunk """ origins: TensorType["num_rays", 3] @@ -168,7 +169,7 @@ class RayBundle(TensorDataclass): nears: Optional[TensorType["num_rays", 1]] = None fars: Optional[TensorType["num_rays", 1]] = None valid_mask: Optional[TensorType["num_rays", 1, bool]] = None - num_rays_per_chunk: Optional[int] = None + num_rays_per_chunk: int = 128 def set_camera_indices(self, camera_index: int) -> None: """Sets all of the the camera indices to a specific camera index. diff --git a/nerfactory/data/structs.py b/nerfactory/data/structs.py index 5c73b31d7..d9b84d255 100644 --- a/nerfactory/data/structs.py +++ b/nerfactory/data/structs.py @@ -41,8 +41,8 @@ class Semantics: stuff_classes: List[str] thing_filenames: List[str] thing_classes: List[str] - stuff_colors: Optional[torch.Tensor] = None - thing_colors: Optional[torch.Tensor] = None + stuff_colors: torch.Tensor + thing_colors: torch.Tensor @dataclass diff --git a/nerfactory/engine/trainer.py b/nerfactory/engine/trainer.py index e11c58e43..a21862942 100644 --- a/nerfactory/engine/trainer.py +++ b/nerfactory/engine/trainer.py @@ -18,7 +18,7 @@ Code to train model. import functools import logging import os -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union import torch import torch.distributed as dist @@ -69,7 +69,7 @@ class Trainer: self.dataloader_train: TrainDataloader self.dataloader_eval: EvalDataloader # model variables - self.graph = Graph + self.graph: Union[Graph, DDP] self.optimizers: Optimizers self.start_step = 0 # logging variables @@ -90,6 +90,8 @@ class Trainer: self.dataset_inputs_train, self.dataloader_train = setup_dataset_train(self.config.data, device=self.device) _, self.dataloader_eval = setup_dataset_eval(self.config.data, test_mode=test_mode, device=self.device) self.graph = setup_graph(self.config.graph, self.dataset_inputs_train, device=self.device) + if not isinstance(self.graph, Graph): + raise ValueError("Graph was improperly initialized.") self.optimizers = setup_optimizers(self.config.optimizers, self.graph.get_param_groups()) self._load_checkpoint() @@ -179,9 +181,9 @@ class Trainer: os.makedirs(output_dir) ckpt_path = os.path.join(output_dir, f"step-{step:09d}.ckpt") if hasattr(self.graph, "module"): - model = self.graph.module.state_dict() + model = self.graph.module.state_dict() # type: ignore else: - model = self.graph.state_dict() + model = self.graph.state_dict() # type: ignore torch.save( { "step": step, @@ -207,7 +209,7 @@ class Trainer: self.optimizers.zero_grad_all() with torch.autocast(device_type=ray_indices.device.type, enabled=self.mixed_precision): _, loss_dict, metrics_dict = self.graph.forward(ray_indices=ray_indices, batch=batch) # type: ignore - loss = sum(loss_dict.values()) + loss = sum(loss_dict.values()) # type: ignore self.grad_scaler.scale(loss).backward() # type: ignore self.optimizers.optimizer_scaler_step_all(self.grad_scaler) self.grad_scaler.update() @@ -217,9 +219,9 @@ class Trainer: callback.after_step(step) # Merging loss and metrics dict into a single output. - loss_dict["loss"] = loss - loss_dict.update(metrics_dict) - return loss_dict + loss_dict["loss"] = loss # type: ignore + loss_dict.update(metrics_dict) # type: ignore + return loss_dict # type: ignore @profiler.time_function def test_image(self, camera_ray_bundle: RayBundle, batch: dict, step: Optional[int] = None) -> float: @@ -240,7 +242,7 @@ class Trainer: outputs = self.graph.get_outputs_for_camera_ray_bundle(camera_ray_bundle) # type: ignore psnr = self.graph.log_test_image_outputs(image_idx, step, batch, outputs) # type: ignore self.graph.train() # type: ignore - return psnr + return psnr # type: ignore def eval_with_dataloader(self, dataloader: EvalDataloader, step: Optional[int] = None) -> None: """Run evaluation with a given dataloader. diff --git a/nerfactory/fields/density_fields/density_grid.py b/nerfactory/fields/density_fields/density_grid.py index bd9409850..d402b6d22 100644 --- a/nerfactory/fields/density_fields/density_grid.py +++ b/nerfactory/fields/density_fields/density_grid.py @@ -15,7 +15,7 @@ """ Code to implement the density grid. """ -from typing import Callable, List, NoReturn, Tuple +from typing import Callable, List, Tuple, Union import torch from torch import nn @@ -25,7 +25,7 @@ import nerfactory.cuda as nerfactory_cuda from nerfactory.cameras.rays import RaySamples -def _create_grid_coords(resolution: int, device: torch.device = "cpu") -> TensorType["n_coords", 3]: +def _create_grid_coords(resolution: int, device: Union[torch.device, str] = "cpu") -> TensorType["n_coords", 3]: """Create 3D grid coordinates Args: @@ -77,6 +77,9 @@ class DensityGrid(nn.Module): update_every_num_iters (int): How frequently to update the grid values. Defaults to 16. """ + density_grid: TensorType["num_cascades", "resolution**3"] + mean_density: TensorType[1] + def __init__( self, center: float = 0.0, @@ -155,7 +158,7 @@ class DensityGrid(nn.Module): step: int, density_threshold: float = 2, # 0.01 / (SQRT3 / 1024 * 3) decay: float = 0.95, - ) -> NoReturn: + ) -> None: """Update the density grid in EMA way. Args: @@ -197,7 +200,7 @@ class DensityGrid(nn.Module): # pack to bitfield self.density_bitfield.data = nerfactory_cuda.packbits( - self.density_grid, min(self.mean_density.item(), density_threshold) + self.density_grid, min(self.mean_density.item(), density_threshold) # type: ignore ) # TODO(ruilongli): max pooling? https://github.com/NVlabs/instant-ngp/blob/master/src/testbed_nerf.cu#L578 diff --git a/nerfactory/fields/instant_ngp_field.py b/nerfactory/fields/instant_ngp_field.py index 891e27979..400ebffd7 100644 --- a/nerfactory/fields/instant_ngp_field.py +++ b/nerfactory/fields/instant_ngp_field.py @@ -153,12 +153,5 @@ class TorchInstantNGPField(NeRFField): ) self.aabb = Parameter(aabb, requires_grad=False) - def get_density(self, ray_samples: RaySamples): - normalized_ray_samples = ray_samples - normalized_ray_samples.positions = SceneBounds.get_normalized_positions( - normalized_ray_samples.frustums.get_positions(), self.aabb - ) - return super().get_density(normalized_ray_samples) - field_implementation_to_class = {"tcnn": TCNNInstantNGPField, "torch": TorchInstantNGPField} diff --git a/nerfactory/fields/nerf_field.py b/nerfactory/fields/nerf_field.py index 3d8e3c8d6..499e02808 100644 --- a/nerfactory/fields/nerf_field.py +++ b/nerfactory/fields/nerf_field.py @@ -84,7 +84,7 @@ class NeRFField(Field): self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim()) self.field_heads = nn.ModuleList(field_heads) for field_head in self.field_heads: - field_head.set_in_dim(self.mlp_head.get_out_dim()) + field_head.set_in_dim(self.mlp_head.get_out_dim()) # type: ignore def get_density(self, ray_samples: RaySamples): if self.use_integrated_encoding: @@ -107,6 +107,6 @@ class NeRFField(Field): outputs = {} for field_head in self.field_heads: encoded_dir = self.direction_encoding(ray_samples.frustums.directions) - mlp_out = self.mlp_head(torch.cat([encoded_dir, density_embedding], dim=-1)) + mlp_out = self.mlp_head(torch.cat([encoded_dir, density_embedding], dim=-1)) # type: ignore outputs[field_head.field_head_name] = field_head(mlp_out) return outputs diff --git a/nerfactory/fields/nerfw_field.py b/nerfactory/fields/nerfw_field.py index f02860c6a..a68451bb7 100644 --- a/nerfactory/fields/nerfw_field.py +++ b/nerfactory/fields/nerfw_field.py @@ -128,11 +128,15 @@ class VanillaNerfWField(Field): """ outputs = {} encoded_dir = self.direction_encoding(ray_samples.frustums.directions) + if ray_samples.camera_indices is None: + raise AttributeError("Camera indices are not provided.") embedded_appearance = self.embedding_appearance(ray_samples.camera_indices.squeeze()) - mlp_head_out = self.mlp_head(torch.cat([density_embedding, encoded_dir, embedded_appearance], dim=-1)) + mlp_in = torch.cat([density_embedding, encoded_dir, embedded_appearance], dim=-1) # type: ignore + mlp_head_out = self.mlp_head(mlp_in) outputs[self.field_head_rgb.field_head_name] = self.field_head_rgb(mlp_head_out) # static rgb embedded_transient = self.embedding_transient(ray_samples.camera_indices.squeeze()) - transient_mlp_out = self.mlp_transient(torch.cat([density_embedding, embedded_transient], dim=-1)) + transient_mlp_in = torch.cat([density_embedding, embedded_transient], dim=-1) # type: ignore + transient_mlp_out = self.mlp_transient(transient_mlp_in) outputs[self.field_head_transient_uncertainty.field_head_name] = self.field_head_transient_uncertainty( transient_mlp_out ) # uncertainty diff --git a/nerfactory/graphs/base.py b/nerfactory/graphs/base.py index 91b412071..955fd8e3f 100644 --- a/nerfactory/graphs/base.py +++ b/nerfactory/graphs/base.py @@ -17,7 +17,7 @@ The Graph module contains all trainable parameters. """ from abc import abstractmethod from collections import defaultdict -from typing import Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union import torch from omegaconf import DictConfig @@ -53,7 +53,9 @@ class AbstractGraph(nn.Module): return self.device_indicator_param.device @abstractmethod - def forward(self, ray_indices: TensorType["num_rays", 3], batch: Union[str, Dict[str, torch.tensor]] = None): + def forward( + self, ray_indices: TensorType["num_rays", 3], batch: Union[str, Optional[Dict[str, torch.Tensor]]] = None + ): """Process starting with ray indices. Turns them into rays, then performs volume rendering.""" @@ -75,14 +77,14 @@ class Graph(AbstractGraph): def __init__( self, - intrinsics: torch.Tensor = None, - camera_to_world: torch.Tensor = None, - loss_coefficients: DictConfig = None, - scene_bounds: SceneBounds = None, + intrinsics: torch.Tensor, + camera_to_world: torch.Tensor, + loss_coefficients: DictConfig, + scene_bounds: Optional[SceneBounds] = None, enable_collider: bool = True, - collider_config: DictConfig = None, + collider_config: Optional[DictConfig] = None, enable_density_field: bool = False, - density_field_config: DictConfig = None, + density_field_config: Optional[DictConfig] = None, **kwargs, ) -> None: super().__init__() @@ -122,6 +124,10 @@ class Graph(AbstractGraph): if self.enable_collider: self.collider = instantiate_from_dict_config(self.collider_config, scene_bounds=self.scene_bounds) + @abstractmethod + def populate_fields(self): + """Set the fields.""" + @abstractmethod def populate_misc_modules(self): """Initializes any additional modules that are part of the network.""" @@ -135,7 +141,7 @@ class Graph(AbstractGraph): """ @abstractmethod - def get_outputs(self, ray_bundle: RayBundle) -> dict: + def get_outputs(self, ray_bundle: RayBundle) -> Dict[str, torch.Tensor]: """Takes in a Ray Bundle and returns a dictionary of outputs. Args: @@ -153,7 +159,9 @@ class Graph(AbstractGraph): v = torch.tile(v, (1, 1, 3)) outputs[k] = v - def forward_after_ray_generator(self, ray_bundle: RayBundle, batch: Union[str, Dict[str, torch.tensor]] = None): + def forward_after_ray_generator( + self, ray_bundle: RayBundle, batch: Optional[Union[str, Dict[str, torch.Tensor]]] = None + ): """Run forward starting with a ray bundle.""" if self.collider is not None: intersected_ray_bundle = self.collider(ray_bundle) @@ -185,19 +193,21 @@ class Graph(AbstractGraph): loss_dict[loss_name] *= self.loss_coefficients[loss_name] return outputs, loss_dict, metrics_dict - def forward(self, ray_indices: TensorType["num_rays", 3], batch: Union[str, Dict[str, torch.tensor]] = None): + def forward( + self, ray_indices: TensorType["num_rays", 3], batch: Optional[Union[str, Dict[str, torch.Tensor]]] = None + ): """Run the forward starting with ray indices.""" ray_bundle = self.ray_generator.forward(ray_indices) return self.forward_after_ray_generator(ray_bundle, batch=batch) - def get_metrics_dict(self, outputs, batch) -> Dict[str, torch.tensor]: + def get_metrics_dict(self, outputs, batch) -> Dict[str, torch.Tensor]: """Compute and returns metrics.""" # pylint: disable=unused-argument # pylint: disable=no-self-use return {} @abstractmethod - def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients) -> Dict[str, torch.tensor]: + def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients) -> Dict[str, torch.Tensor]: """Computes and returns the losses dict.""" @torch.no_grad() @@ -213,10 +223,10 @@ class Graph(AbstractGraph): end_idx = i + camera_ray_bundle.num_rays_per_chunk ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(start_idx, end_idx) outputs = self.forward_after_ray_generator(ray_bundle) - for output_name, output in outputs.items(): + for output_name, output in outputs.items(): # type: ignore outputs_lists[output_name].append(output) for output_name, outputs_list in outputs_lists.items(): - outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) + outputs[output_name] = torch.cat(outputs_list).view(image_height, image_width, -1) # type: ignore return outputs def get_outputs_for_camera(self, camera: Camera): @@ -230,7 +240,8 @@ class Graph(AbstractGraph): def load_graph(self, loaded_state: Dict[str, Any]) -> None: """Load the checkpoint from the given path""" - self.load_state_dict({key.replace("module.", ""): value for key, value in loaded_state["model"].items()}) + state = {key.replace("module.", ""): value for key, value in loaded_state["model"].items()} + self.load_state_dict(state) # type: ignore @profiler.time_function diff --git a/nerfactory/graphs/instant_ngp.py b/nerfactory/graphs/instant_ngp.py index b2103b008..d0c63960e 100644 --- a/nerfactory/graphs/instant_ngp.py +++ b/nerfactory/graphs/instant_ngp.py @@ -55,13 +55,15 @@ class NGPGraph(Graph): Callback( update_every_num_iters=self.density_field.update_every_num_iters, func=self.density_field.update_density_grid, - density_eval_func=self.field.density_fn, + density_eval_func=self.field.density_fn, # type: ignore ) ] def populate_fields(self): """Set the fields.""" # torch or tiny-cuda-nn version + if self.scene_bounds is None: + raise ValueError("scene_bounds must be set to use an NGPGraph") self.field = field_implementation_to_class[self.field_implementation](self.scene_bounds.aabb) def populate_misc_modules(self): @@ -78,6 +80,8 @@ class NGPGraph(Graph): def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} + if self.field is None: + raise ValueError("populate_fields() must be called before get_param_groups") param_groups["fields"] = list(self.field.parameters()) return param_groups @@ -89,6 +93,8 @@ class NGPGraph(Graph): num_rays = len(ray_bundle) device = ray_bundle.origins.device + if self.field is None: + raise ValueError("populate_fields() must be called before get_outputs") ray_samples, packed_info, t_min, t_max = self.sampler(ray_bundle, self.field.aabb) field_outputs = self.field.forward(ray_samples) @@ -170,7 +176,7 @@ class NGPGraph(Graph): lpips = self.lpips(image, rgb) writer.put_scalar(name=f"psnr/val_{image_idx}-fine", scalar=float(psnr), step=step) - writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(ssim), step=step) + writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(ssim), step=step) # type: ignore writer.put_scalar(name=f"lpips/val_{image_idx}", scalar=float(lpips), step=step) writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(psnr), step=step) diff --git a/nerfactory/graphs/mipnerf.py b/nerfactory/graphs/mipnerf.py index 68fb07683..e3b1a5f8f 100644 --- a/nerfactory/graphs/mipnerf.py +++ b/nerfactory/graphs/mipnerf.py @@ -94,10 +94,16 @@ class MipNerfGraph(Graph): def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} + if self.field is None: + raise ValueError("populate_fields() must be called before get_param_groups") param_groups["fields"] = list(self.field.parameters()) return param_groups def get_outputs(self, ray_bundle: RayBundle): + + if self.field is None: + raise ValueError("populate_fields() must be called before get_outputs") + # uniform sampling ray_samples_uniform = self.sampler_uniform(ray_bundle) @@ -183,7 +189,7 @@ class MipNerfGraph(Graph): writer.put_scalar(name=f"psnr/val_{image_idx}-coarse", scalar=float(coarse_psnr), step=step) writer.put_scalar(name=f"psnr/val_{image_idx}-fine", scalar=float(fine_psnr), step=step) - writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(fine_ssim), step=step) + writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(fine_ssim), step=step) # type: ignore writer.put_scalar(name=f"lpips/val_{image_idx}", scalar=float(fine_lpips), step=step) writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(fine_psnr), step=step) diff --git a/nerfactory/graphs/mipnerf_360.py b/nerfactory/graphs/mipnerf_360.py index 447ebdad6..e656039b4 100644 --- a/nerfactory/graphs/mipnerf_360.py +++ b/nerfactory/graphs/mipnerf_360.py @@ -99,10 +99,16 @@ class MipNerf360Graph(Graph): def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} + if self.field is None: + raise ValueError("populate_fields() must be called before get_param_groups") param_groups["fields"] = list(self.field.parameters()) return param_groups def get_outputs(self, ray_bundle: RayBundle): + + if self.field is None: + raise ValueError("populate_fields() must be called before get_outputs") + # uniform sampling ray_samples_uniform = self.sampler_uniform(ray_bundle) @@ -143,7 +149,7 @@ class MipNerf360Graph(Graph): } return outputs - def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients) -> Dict[str, torch.tensor]: + def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients) -> Dict[str, torch.Tensor]: image = batch["image"] rgb_loss_coarse = self.rgb_loss(image, outputs["rgb_coarse"]) rgb_loss_fine = self.rgb_loss(image, outputs["rgb_fine"]) @@ -195,7 +201,7 @@ class MipNerf360Graph(Graph): writer.put_scalar(name=f"psnr/val_{image_idx}-coarse", scalar=float(coarse_psnr), step=step) writer.put_scalar(name=f"psnr/val_{image_idx}-fine", scalar=float(fine_psnr), step=step) - writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(fine_ssim), step=step) + writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(fine_ssim), step=step) # type: ignore writer.put_scalar(name=f"lpips/val_{image_idx}", scalar=float(fine_lpips), step=step) writer.put_scalar( name=f"ray_loss_coarse/val_{image_idx}", scalar=float(torch.mean(outputs["ray_loss_coarse"])), step=step diff --git a/nerfactory/graphs/modules/ray_sampler.py b/nerfactory/graphs/modules/ray_sampler.py index b293e0b60..313d16755 100644 --- a/nerfactory/graphs/modules/ray_sampler.py +++ b/nerfactory/graphs/modules/ray_sampler.py @@ -84,7 +84,7 @@ class SpacedSampler(Sampler): @torch.no_grad() def generate_ray_samples( self, - ray_bundle: RayBundle = None, + ray_bundle: Optional[RayBundle] = None, num_samples: Optional[int] = None, ) -> RaySamples: """Generates position samples accoring to spacing function. @@ -236,7 +236,7 @@ class PDFSampler(Sampler): num_samples: int, train_stratified: bool = True, include_original: bool = True, - density_field: DensityGrid = None, + density_field: Optional[DensityGrid] = None, weight_threshold: float = 1e-2, histogram_padding: float = 0.01, ) -> None: @@ -257,8 +257,8 @@ class PDFSampler(Sampler): @torch.no_grad() def generate_ray_samples( self, - ray_bundle: RayBundle = None, - ray_samples: RaySamples = None, + ray_bundle: Optional[RayBundle] = None, + ray_samples: Optional[RaySamples] = None, weights: TensorType[..., "num_samples", 1] = None, num_samples: Optional[int] = None, eps: float = 1e-5, @@ -275,6 +275,10 @@ class PDFSampler(Sampler): Returns: Positions and deltas for samples along a ray """ + + if ray_samples is None or ray_bundle is None: + raise ValueError("ray_samples and ray_bundle must be provided") + num_samples = num_samples or self.num_samples num_bins = num_samples + 1 @@ -370,6 +374,10 @@ class NGPSpacedSampler(Sampler): Second return contains all the information to recover packed samples into unpacked mode for rendering. The last two returns are t_min and t_max from ray-aabb test. """ + + if self.density_field is None: + raise ValueError("density_field must be set to use NGPSpacedSampler") + num_samples = num_samples or self.num_samples aabb = aabb.flatten() @@ -380,8 +388,8 @@ class NGPSpacedSampler(Sampler): if t_min is None or t_max is None: # TODO(ruilongli): this clipping here is stupid. Try to avoid that. t_min, t_max = nerfactory_cuda.ray_aabb_intersect(rays_o, rays_d, aabb) - t_min = torch.clamp(t_min, max=1e10) - t_max = torch.clamp(t_max, max=1e10) + t_min = torch.clamp(t_min, max=1e10) # type: ignore + t_max = torch.clamp(t_max, max=1e10) # type: ignore marching_steps = -1 # disable marching mode for now # TODO(ruilongli): * 16 is for original impl for training. Need to run diff --git a/nerfactory/graphs/nerfw.py b/nerfactory/graphs/nerfw.py index c6f01fbf4..7a872baee 100644 --- a/nerfactory/graphs/nerfw.py +++ b/nerfactory/graphs/nerfw.py @@ -113,10 +113,16 @@ class NerfWGraph(Graph): def get_param_groups(self): param_groups = {} + if self.field_coarse is None or self.field_fine is None: + raise ValueError("populate_fields() must be called before get_param_groups") param_groups["fields"] = list(self.field_coarse.parameters()) + list(self.field_fine.parameters()) return param_groups def get_outputs(self, ray_bundle: RayBundle): + + if self.field_coarse is None or self.field_fine is None: + raise ValueError("populate_fields() must be called before get_outputs") + # uniform sampling ray_samples_uniform = self.sampler_uniform(ray_bundle) diff --git a/nerfactory/graphs/semantic_nerf.py b/nerfactory/graphs/semantic_nerf.py index 95d9f7d66..f8212deaf 100644 --- a/nerfactory/graphs/semantic_nerf.py +++ b/nerfactory/graphs/semantic_nerf.py @@ -90,7 +90,7 @@ class SemanticNerfField(Field): self, ray_samples: RaySamples, density_embedding: Optional[TensorType] = None ) -> Dict[FieldHeadNames, TensorType]: encoded_dir = self.direction_encoding(ray_samples.frustums.directions) - mlp_out = self.mlp_head(torch.cat([encoded_dir, density_embedding], dim=-1)) + mlp_out = self.mlp_head(torch.cat([encoded_dir, density_embedding], dim=-1)) # type: ignore outputs = {} # rgb outputs[self.field_head_rgb.field_head_name] = self.field_head_rgb(mlp_out) @@ -103,7 +103,7 @@ class SemanticNerfField(Field): class SemanticNerfGraph(NeRFGraph): """Semantic-NeRF graph""" - def __init__(self, intrinsics=None, camera_to_world=None, semantics: Semantics = None, **kwargs) -> None: + def __init__(self, intrinsics, camera_to_world, semantics: Semantics, **kwargs) -> None: self.stuff_classes = semantics.stuff_classes self.stuff_colors = semantics.stuff_colors super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs) @@ -185,7 +185,7 @@ class SemanticNerfGraph(NeRFGraph): def log_test_image_outputs(self, image_idx, step, batch, outputs): super().log_test_image_outputs(image_idx, step, batch, outputs) semantic_logits = outputs["semantic_fine"] - semantic_labels = torch.argmax(torch.nn.functional.softmax(semantic_logits, dim=-1), dim=-1) + semantic_labels = torch.argmax(torch.nn.functional.softmax(semantic_logits, dim=-1), dim=-1) # type: ignore semantic_labels_image = self.stuff_colors[semantic_labels] writer.put_image(name=f"semantics/image_idx_{image_idx}", image=semantic_labels_image, step=step) diff --git a/nerfactory/graphs/vanilla_nerf.py b/nerfactory/graphs/vanilla_nerf.py index 3e2e687fd..481004ec1 100644 --- a/nerfactory/graphs/vanilla_nerf.py +++ b/nerfactory/graphs/vanilla_nerf.py @@ -17,7 +17,7 @@ Implementation of vanilla nerf. """ -from typing import Dict, List +from typing import Dict, List, Optional import torch from omegaconf import DictConfig @@ -58,14 +58,14 @@ class NeRFGraph(Graph): def __init__( self, - intrinsics: torch.Tensor = None, - camera_to_world: torch.Tensor = None, + intrinsics: torch.Tensor, + camera_to_world: torch.Tensor, near_plane: float = 2.0, far_plane: float = 6.0, num_coarse_samples: int = 64, num_importance_samples: int = 128, enable_density_field: bool = False, - density_field_config: DictConfig = None, + density_field_config: Optional[DictConfig] = None, **kwargs, ) -> None: self.near_plane = near_plane @@ -83,6 +83,9 @@ class NeRFGraph(Graph): ) def get_training_callbacks(self) -> List[Callback]: + if self.field_coarse is None: + raise ValueError("populate fields must be called before get_training_callbacks.") + callbacks = [] if self.density_field is not None: callbacks = [ @@ -127,10 +130,16 @@ class NeRFGraph(Graph): def get_param_groups(self) -> Dict[str, List[Parameter]]: param_groups = {} + if self.field_coarse is None or self.field_fine is None: + raise ValueError("populate_fields() must be called before get_param_groups") param_groups["fields"] = list(self.field_coarse.parameters()) + list(self.field_fine.parameters()) return param_groups def get_outputs(self, ray_bundle: RayBundle): + + if self.field_coarse is None or self.field_fine is None: + raise ValueError("populate_fields() must be called before get_outputs") + # uniform sampling ray_samples_uniform = self.sampler_uniform(ray_bundle) @@ -167,7 +176,7 @@ class NeRFGraph(Graph): } return outputs - def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients) -> Dict[str, torch.tensor]: + def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients) -> Dict[str, torch.Tensor]: # Scaling metrics by coefficients to create the losses. device = outputs["rgb_coarse"].device image = batch["image"].to(device) @@ -218,7 +227,7 @@ class NeRFGraph(Graph): writer.put_scalar(name=f"psnr/val_{image_idx}-coarse", scalar=float(coarse_psnr), step=step) writer.put_scalar(name=f"psnr/val_{image_idx}-fine", scalar=float(fine_psnr), step=step) - writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(fine_ssim), step=step) + writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(fine_ssim), step=step) # type: ignore writer.put_scalar(name=f"lpips/val_{image_idx}", scalar=float(fine_lpips), step=step) writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(fine_psnr), step=step) diff --git a/nerfactory/utils/io.py b/nerfactory/utils/io.py index aaa18d33d..7a0e89fd7 100644 --- a/nerfactory/utils/io.py +++ b/nerfactory/utils/io.py @@ -19,7 +19,7 @@ import json import logging import os import pickle -from typing import Any, Optional, Tuple +from typing import Any, Tuple def load_from_json(filename: str): @@ -68,7 +68,7 @@ def write_to_pkl(filename: str, content: Any): pickle.dump(content, f) -def get_git_root(path: str, dirs: Tuple[str] = (".git",), default=None) -> Optional[str]: +def get_git_root(path: str, dirs: Tuple[str] = (".git",), default: str = "") -> str: """Find the root of the git repository. 'https://stackoverflow.com/questions/22081209/find-the-root-of-the-git-repository-where-the-file-lives' @@ -85,15 +85,19 @@ def get_git_root(path: str, dirs: Tuple[str] = (".git",), default=None) -> Optio if any(os.path.isdir(os.path.join(test, directory)) for directory in dirs): return test prev, test = test, os.path.abspath(os.path.join(test, os.pardir)) + logging.info("Couldn't find git root") return default -def get_project_root(path: str): +def get_project_root(path: str) -> str: """Return the project root directory from an environment variable. # TODO: handle this better and report user error Args: path: The path to start from. + + Returns: + The project root directory. """ project_root = os.getenv("PROJECT_ROOT") if project_root is None: diff --git a/nerfactory/utils/plotly.py b/nerfactory/utils/plotly.py index 5722b0b61..811c5c8e7 100644 --- a/nerfactory/utils/plotly.py +++ b/nerfactory/utils/plotly.py @@ -21,11 +21,12 @@ The function use prefix conventions in the following way: return 'go.Figure' objects which are the plots. Go Figure! :') """ -from typing import List, Union +from typing import Any, List, Optional, Union import numpy as np import plotly.graph_objects as go import torch +import torch.nn.functional as F from plotly import express as ex from torchtyping import TensorType @@ -51,12 +52,12 @@ def get_line_segments_from_lines( lines: TensorType["num_rays", 2, 3], color: str = color_str((1, 0, 0)), marker_color: str = color_str((1, 0, 0)), - colors: List[str] = None, + colors: Optional[List[str]] = None, draw_marker: bool = True, draw_line: bool = True, marker_size: float = 4, line_width: float = 10, -) -> go.Scatter3d: +) -> List[Any]: """Returns a list of Scatter3D objects for creating lines with plotly. # TODO(ethan): make this function more efficient instead of having a list of objects. @@ -98,7 +99,7 @@ def get_line_segments_from_lines( return data -def vis_dataset(camera_origins: TensorType["num_cameras", 3], ray_bundle: RayBundle) -> go.FigureWidget: +def vis_dataset(camera_origins: TensorType["num_cameras", 3], ray_bundle: RayBundle) -> go.FigureWidget: # type: ignore """Visualize a dataset with plotly using our cameras and generated rays. Args: @@ -136,8 +137,8 @@ def vis_dataset(camera_origins: TensorType["num_cameras", 3], ray_bundle: RayBun autosize=False, width=1000, height=1000, - margin=go.layout.Margin(l=50, r=50, b=100, t=100, pad=4), - scene=go.layout.Scene( + margin=go.layout.Margin(l=50, r=50, b=100, t=100, pad=4), # type: ignore + scene=go.layout.Scene( # type: ignore aspectmode="data", camera=dict(up=dict(x=0, y=0, z=1), center=dict(x=0, y=0, z=0), eye=dict(x=1.25, y=1.25, z=1.25)), ), @@ -146,7 +147,7 @@ def vis_dataset(camera_origins: TensorType["num_cameras", 3], ray_bundle: RayBun return fig -def get_random_color(colormap: List[str] = None, idx: int = None) -> str: +def get_random_color(colormap: Optional[List[str]] = None, idx: Optional[int] = None) -> str: """Get a random color from a colormap. Args: @@ -165,7 +166,7 @@ def get_random_color(colormap: List[str] = None, idx: int = None) -> str: def get_sphere( radius: float, center: TensorType[3] = None, color: str = "black", opacity: float = 1.0, resolution: int = 32 -) -> go.Mesh3d: +) -> go.Mesh3d: # type: ignore """Returns a sphere object for plotting with plotly. Args: @@ -211,7 +212,7 @@ def get_gaussian_ellipsiod( opacity: float = 0.5, resolution: int = 20, name: str = "ellipse", -) -> go.Mesh3d: +) -> go.Mesh3d: # type: ignore """Get a plotly ellipsoid for a Gaussian. Args: @@ -264,7 +265,7 @@ def get_gaussian_ellipsiod( def get_gaussian_ellipsoids_list( gaussians: Gaussians, opacity: float = 0.5, color: str = "random", resolution: int = 20 -) -> List[Union[go.Mesh3d, go.Scatter3d]]: +) -> List[Union[go.Mesh3d, go.Scatter3d]]: # type: ignore """Get a list of plotly meshes for frustums. Args: @@ -307,7 +308,7 @@ def get_gaussian_ellipsoids_list( def get_frustum_mesh( frustum: Frustums, opacity: float = 0.3, color: str = "#DC203C", resolution: int = 20 -) -> go.Mesh3d: +) -> go.Mesh3d: # type: ignore """Get a plotly mesh for a single frustum. Args: @@ -338,8 +339,8 @@ def get_frustum_mesh( pts = torch.stack([x, y, z], dim=-1) forward = frustum.directions - up = torch.nn.functional.normalize(torch.cross(forward, torch.tensor([0.0, 0, 1])), dim=-1) - right = torch.nn.functional.normalize(torch.cross(forward, up), dim=-1) + up = F.normalize(torch.cross(forward, torch.tensor([0.0, 0, 1])), dim=-1) # type: ignore + right = F.normalize(torch.cross(forward, up), dim=-1) # type: ignore rotation = torch.stack([forward, up, right], dim=1) pts = torch.einsum("kj,ij->ki", pts, rotation) @@ -358,7 +359,7 @@ def get_frustum_mesh( def get_frustums_mesh_list( frustums: Frustums, opacity: float = 1.0, color: str = "random", resolution: int = 20 -) -> List[go.Mesh3d]: +) -> List[go.Mesh3d]: # type: ignore """Get a list of plotly meshes for a list of frustums. Args: @@ -371,7 +372,7 @@ def get_frustums_mesh_list( List of plotly meshes """ data = [] - for i, frustum in enumerate(frustums.flatten()): + for i, frustum in enumerate(frustums.flatten()): # type: ignore if color == "random": c = get_random_color(idx=i) else: @@ -382,7 +383,7 @@ def get_frustums_mesh_list( def get_frustum_points( frustum: Frustums, opacity: float = 1.0, color: str = "forestgreen", size: float = 5 -) -> go.Scatter3d: +) -> go.Scatter3d: # type: ignore """Get a set plotly points for frustums centers. Args: @@ -414,7 +415,7 @@ def get_frustum_points( def get_ray_bundle_lines( ray_bundle: RayBundle, length: float = 1.0, color: str = "#DC203C", width: float = 1 -) -> go.Scatter3d: +) -> go.Scatter3d: # type: ignore """Get a plotly line for a ray bundle. Args: @@ -443,7 +444,7 @@ def get_ray_bundle_lines( ) -def vis_camera_rays(camera: Camera) -> go.Figure: +def vis_camera_rays(camera: Camera) -> go.Figure: # type: ignore """Visualize camera rays. Args: diff --git a/nerfactory/utils/visualization.py b/nerfactory/utils/visualization.py index e4183293e..fcaa777cc 100644 --- a/nerfactory/utils/visualization.py +++ b/nerfactory/utils/visualization.py @@ -35,7 +35,7 @@ def apply_colormap(image: TensorType["bs":..., 1], cmap="viridis") -> TensorType """ colormap = cm.get_cmap(cmap) - colormap = torch.tensor(colormap.colors).to(image.device) + colormap = torch.tensor(colormap.colors).to(image.device) # type: ignore image_long = (image * 255).long() image_long_min = torch.min(image_long) image_long_max = torch.max(image_long) @@ -64,8 +64,8 @@ def apply_depth_colormap( Colored depth image """ - near_plane = near_plane or torch.min(depth) - far_plane = far_plane or torch.max(depth) + near_plane = near_plane or float(torch.min(depth)) + far_plane = far_plane or float(torch.max(depth)) depth = (depth - near_plane) / (far_plane - near_plane + 1e-10) depth = torch.clip(depth, 0, 1)