diff --git a/nerfactory/cameras/cameras.py b/nerfactory/cameras/cameras.py index 6c3c5b499..f1cf9f9c7 100644 --- a/nerfactory/cameras/cameras.py +++ b/nerfactory/cameras/cameras.py @@ -15,9 +15,11 @@ """ Camera Models """ +import base64 from abc import abstractmethod -from typing import Optional, Type +from typing import Dict, Optional, Type +import cv2 import torch from torch.nn.functional import normalize from torchtyping import TensorType @@ -27,12 +29,15 @@ from nerfactory.utils.misc import is_not_none class Camera: - """Base Camera. Intended to be subclassed""" + """Base Camera. Intended to be subclassed. + + Args: + camera_to_world: Camera to world transformation + camera_index: Camera index + """ def __init__( - self, - camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3], - camera_index: int = None, + self, camera_to_world: TensorType[3, 4] = torch.eye(4)[:3], camera_index: Optional[int] = None ) -> None: self.camera_to_world = camera_to_world self.camera_index = camera_index @@ -46,7 +51,7 @@ class Camera: def get_num_intrinsics_params(self) -> int: """ Returns: - int: number of optimizable intrinsic parameters + number of optimizable intrinsic parameters """ return @@ -54,21 +59,21 @@ class Camera: def get_intrinsics(self) -> torch.Tensor: """ Returns: - torch.Tensor: Intrinsics matrix + Intrinsics matrix """ return def get_camera_to_world(self) -> TensorType[3, 4]: """ Returns: - TensorType[3, 4]: Camera to world transformation + Camera to world transformation """ return self.camera_to_world def get_camera_to_world_h(self) -> TensorType[4, 4]: """ Returns: - TensorType[4, 4]: Camera to world transformation with homogeneous coordinates + Camera to world transformation with homogeneous coordinates """ c2w = self.camera_to_world ones = torch.tensor([0, 0, 0, 1], device=c2w.device)[None] @@ -79,7 +84,7 @@ class Camera: def get_image_height(self) -> int: """ Returns: - int: Image height + Image height """ return @@ -87,7 +92,7 @@ class Camera: def get_image_width(self) -> int: """ Returns: - int: Image width + Image width """ return @@ -96,7 +101,7 @@ class Camera: """Rescales the camera intrinsics for output resolution. Args: - scaling_factor (float): Scaling factor + scaling_factor: Scaling factor Returns: None @@ -156,9 +161,42 @@ class Camera: ray_bundle.set_camera_indices(camera_index=self.camera_index) return ray_bundle + @abstractmethod + def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict: + """Converts the camera to a json dictionary. + + Args: + image: An image that is encoded to a base64 string. Defaults to None. + + Returns: + A JSON representation of the camera + """ + raise NotImplementedError + + @staticmethod + def from_json(json_: Dict) -> "Camera": # pylint:disable=no-self-use + """Converts a json dictionary to the camera object. + + Args: + json_: The json dictionary. + + Returns: + The camera object instantiated from the json dictionary. + """ + raise NotImplementedError + class PinholeCamera(Camera): - """Pinhole camera model.""" + """Pinhole camera model. + + Args: + fx: Focal length in x direction + fy: Focal length in y direction + cx: Principal point in x direction + cy: Principal point in y direction + camera_to_world: Camera to world transformation + camera_index: Camera index. Defaults to None. + """ def __init__( self, @@ -166,10 +204,10 @@ class PinholeCamera(Camera): cy: float, fx: float, fy: float, - camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3], - camera_index: int = None, + camera_to_world: TensorType[3, 4] = torch.eye(4)[:3], + camera_index: Optional[int] = None, ): - super().__init__(camera_to_world, camera_index) + super().__init__(camera_to_world=camera_to_world, camera_index=camera_index) self.cx = cx self.cy = cy self.fx = fx @@ -258,19 +296,48 @@ class PinholeCamera(Camera): return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None]) + def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict: + json_ = { + "type": "PinholeCamera", + "cx": self.cx, + "cy": self.cy, + "fx": self.fx, + "fy": self.fy, + "camera_to_world": self.camera_to_world.tolist(), + "camera_index": self.camera_index, + } + if image: + # move image to cpu if not already + # TODO: move to numpy + data = cv2.imencode(".png", image[:, :, ::-1])[1].tobytes() + json_["image"] = str("data:image/png;base64," + base64.b64encode(data).decode("ascii")) + return json_ + + @staticmethod + def from_json(json_: Dict) -> "PinholeCamera": # pylint:disable=no-self-use + raise NotImplementedError + class SimplePinholeCamera(PinholeCamera): - """Simple Pinhole Camera model.""" + """Simple Pinhole Camera model. + + Args: + cx: Principal point x direction + cy: Principal point y direction + f: Focal length + camera_to_world: Camera to world transformation matrix + camera_index: Camera index. Defaults to None. + """ def __init__( self, cx: float, cy: float, f: float, - camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3], - camera_index: int = None, + camera_to_world: TensorType[3, 4] = torch.eye(4)[:3], + camera_index: Optional[int] = None, ): - super().__init__(cx, cy, f, f, camera_to_world, camera_index) + super().__init__(cx, cy, f, f, camera_to_world=camera_to_world, camera_index=camera_index) @classmethod def fx_index(cls): @@ -280,18 +347,32 @@ class SimplePinholeCamera(PinholeCamera): def fy_index(cls): return 2 + def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict: + raise NotImplementedError + + @staticmethod + def from_json(json_: Dict) -> "SimplePinholeCamera": # pylint:disable=no-self-use + raise NotImplementedError + class EquirectangularCamera(Camera): - """Equirectangular (360 degree) camera model.""" + """Equirectangular (360 degree) camera model. + + Args: + height: Height of the image + width: Width of the image + camera_to_world: Camera to world transformation matrix + camera_index: Camera index. Defaults to None. + """ def __init__( self, height: int, width: int, - camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3], - camera_index: int = None, + camera_to_world: TensorType[3, 4] = torch.eye(4)[:3], + camera_index: Optional[int] = None, ): - super().__init__(camera_to_world, camera_index) + super().__init__(camera_to_world=camera_to_world, camera_index=camera_index) self.height = height self.width = width @@ -361,6 +442,13 @@ class EquirectangularCamera(Camera): pixel_area = dx * dy return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None]) + def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict: + raise NotImplementedError + + @staticmethod + def from_json(json_: Dict) -> "SimplePinholeCamera": # pylint:disable=no-self-use + raise NotImplementedError + def get_intrinsics_from_intrinsics_matrix(intrinsics_matrix: TensorType[3, 3]): """ @@ -376,6 +464,8 @@ def get_intrinsics_from_intrinsics_matrix(intrinsics_matrix: TensorType[3, 3]): def get_camera_model(num_intrinsics_params: int) -> Type[Camera]: + # TODO: we should specify the type in the dataloading pipeline instead of relying on number of intrinsics + # to make the choice """Returns the camera model given the specified number of intrinsics parameters. Args: diff --git a/nerfactory/cameras/rays.py b/nerfactory/cameras/rays.py index 8a032d962..a1a9b7a5b 100644 --- a/nerfactory/cameras/rays.py +++ b/nerfactory/cameras/rays.py @@ -98,9 +98,9 @@ class RaySamples(TensorDataclass): """ frustums: Frustums - camera_indices: TensorType["bs":..., 1] = None - valid_mask: TensorType["bs":..., 1] = None - deltas: TensorType["bs":..., 1] = None + camera_indices: Optional[TensorType["bs":..., 1]] = None + valid_mask: Optional[TensorType["bs":..., 1]] = None + deltas: Optional[TensorType["bs":..., 1]] = None def get_weights(self, densities: TensorType[..., "num_samples", 1]) -> TensorType[..., "num_samples", 1]: """Return weights based on predicted densities @@ -115,6 +115,7 @@ class RaySamples(TensorDataclass): delta_density = self.deltas * densities alphas = 1 - torch.exp(-delta_density) + # TODO: put both versions of the transmittance calculation somewhere # mip-nerf version of transmittance calculation: transmittance = torch.cumsum(delta_density[..., :-1, :], dim=-2) transmittance = torch.cat( @@ -170,17 +171,6 @@ class RayBundle(TensorDataclass): valid_mask: Optional[TensorType["num_rays", 1, bool]] = None num_rays_per_chunk: int = None - def move_to_device(self, device: torch.device) -> None: - """Move bundle data to a device. - - Args: - device (torch.device): Device to move tensors to. - """ - self.origins = self.origins.to(device) - self.directions = self.directions.to(device) - if is_not_none(self.camera_indices): - self.camera_indices = self.camera_indices.to(device) - def set_camera_indices(self, camera_index: int) -> None: """Sets all of the the camera indices to a specific camera index. diff --git a/nerfactory/cuda/csrc/morton3d.cu b/nerfactory/cuda/csrc/morton3d.cu index 5dd864f1f..68580faa9 100644 --- a/nerfactory/cuda/csrc/morton3d.cu +++ b/nerfactory/cuda/csrc/morton3d.cu @@ -51,6 +51,7 @@ __global__ void kernel_morton3D_invert( torch::Tensor morton3D(const torch::Tensor coords){ DEVICE_GUARD(coords); CHECK_INPUT(coords); + // NOTE(ethan): why & and not &&? TORCH_CHECK(coords.ndimension() == 2 & coords.size(1) == 3) const int N = coords.size(0); @@ -75,7 +76,7 @@ torch::Tensor morton3D(const torch::Tensor coords){ } /** - * @brief Convert 3D indices to coords in z-order + * @brief Convert indices in z-order to 3D coords * * @param indices Integer indices with shape [N] * @return torch::Tensor Integer coords with shape [N, 3] diff --git a/nerfactory/data/structs.py b/nerfactory/data/structs.py index 437d656cd..733bd5343 100644 --- a/nerfactory/data/structs.py +++ b/nerfactory/data/structs.py @@ -17,7 +17,7 @@ Dataset input structures. """ from dataclasses import dataclass -from typing import List, Optional +from typing import Dict, List, Optional import torch from torchtyping import TensorType @@ -27,7 +27,7 @@ from nerfactory.cameras.rays import RayBundle @dataclass class PointCloud: - """_summary_""" + """Dataclass for a point cloud.""" xyz: TensorType["num_points", 3] = None rgb: TensorType["num_points", 3] = None @@ -35,22 +35,24 @@ class PointCloud: @dataclass class Semantics: - """_summary_""" + """Dataclass for semantic labels.""" - stuff_classes: List[str] = None - stuff_colors: List[List[int]] = None - stuff_filenames: List[str] = None - thing_classes: List[str] = None - thing_colors: List[List[int]] = None - thing_filenames: List[str] = None + stuff_classes: Optional[List[str]] = None + stuff_colors: Optional[List[str]] = None + stuff_filenames: Optional[List[str]] = None + thing_classes: Optional[List[str]] = None + thing_colors: Optional[List[str]] = None + thing_filenames: Optional[List[str]] = None @dataclass class SceneBounds: """Data to represent the scene bounds. - aabb: axis-aligned bounding box - aabb[0] is the minimum (x,y,z) point. aabb[1] is the maximum (x,y,z) point. + Attributes: + aabb: axis-aligned bounding box. + aabb[0] is the minimum (x,y,z) point. + aabb[1] is the maximum (x,y,z) point. """ aabb: TensorType[2, 3] = None @@ -71,6 +73,32 @@ class SceneBounds: about the origin.""" return SceneBounds(aabb=(self.aabb - self.get_center()) * scale_factor) + @staticmethod + def get_normalized_positions(positions: TensorType[..., 3], aabb: TensorType[2, 3]): + """Return normalized positions in range [0, 1] based on the aabb axis-aligned bounding box. + + Args: + positions: the xyz positions + aabb: the axis-aligned bounding box + + Returns: + positions that are normalized into the range [0, 1]. + """ + aabb_lengths = aabb[1] - aabb[0] + normalized_positions = (positions - aabb[0]) / aabb_lengths + return normalized_positions + + def to_json(self) -> Dict: + """Returns a json object from the Python object.""" + return {"type": "aabb", "min_point": self.aabb[0].tolist(), "max_point": self.aabb[1].tolist()} + + @staticmethod + def from_json(json_: Dict) -> "SceneBounds": + """Returns the an instance of SceneBounds from a json dictionary.""" + assert json_["type"] == "aabb" + aabb = torch.tensor([json_[0], json_[1]]) + return SceneBounds(aabb=aabb) + @dataclass class DatasetInputs: diff --git a/nerfactory/engine/trainer.py b/nerfactory/engine/trainer.py index 132355dfb..ca78da24d 100644 --- a/nerfactory/engine/trainer.py +++ b/nerfactory/engine/trainer.py @@ -44,7 +44,7 @@ logging.getLogger("PIL").setLevel(logging.WARNING) class Trainer: - """Training class + """Trainer class Args: config (Config): The configuration object. @@ -74,8 +74,9 @@ class Trainer: profiler.setup_profiler(config.logging) # visualizer variable self.visualizer_state = viewer_utils.VisualizerState(config.viewer) - self.grad_scaler = GradScaler(enabled=self.mixed_precision) + # training callbacks + self.callbacks = None def setup(self, test_mode=False): """Setup the Trainer by calling other setup functions. @@ -95,7 +96,7 @@ class Trainer: self.graph = DDP(self.graph, device_ids=[self.local_rank]) dist.barrier(device_ids=[self.local_rank]) - self.graph.register_callbacks() + self.callbacks = self.graph.get_training_callbacks() @classmethod def get_aggregated_loss(cls, loss_dict: Dict[str, torch.Tensor]) -> torch.Tensor: @@ -203,9 +204,8 @@ class Trainer: self.grad_scaler.update() self.optimizers.scheduler_step_all(step) - if self.graph.callbacks: - for func_ in self.graph.callbacks: - func_.after_step(step) + for callback in self.callbacks: + callback.after_step(step) # Merging loss and metrics dict into a single output. loss_dict["loss"] = loss diff --git a/nerfactory/fields/density_fields/density_grid.py b/nerfactory/fields/density_fields/density_grid.py index b235547aa..72cd53a23 100644 --- a/nerfactory/fields/density_fields/density_grid.py +++ b/nerfactory/fields/density_fields/density_grid.py @@ -30,10 +30,10 @@ def _create_grid_coords(resolution: int, device: torch.device = "cpu") -> Tensor Args: resolution (int): The 3D resolution of the grid. - device (torch.device): Which device you want the returned tensor to live int. + device (torch.device): Which device you want the returned tensor to live on. Returns: - TensorType["n_coords", 3]: All grid coordinates with shape [res * res * res, 3] + TensorType["n_coords", 3]: All grid coordinates with shape [res * res * res, 3]. """ arrange = torch.arange(resolution, device=device) grid_x, grid_y, grid_z = torch.meshgrid([arrange, arrange, arrange], indexing="ij") @@ -51,9 +51,9 @@ class DensityGrid(nn.Module): on the scale but all shares the same center. For example: - - `center=0`, `base_scale=3`, `num_cascades=1` would create one grid (lvl=0) covers [-1.5, -1.5, -1.5, 1.5, 1.5, 1.5]. - - `center=0`, `base_scale=3`, `num_cascades=2` would create one grid (lvl=0) covers [-1.5, -1.5, -1.5, 1.5, 1.5, 1.5], - and one more grid (lvl=1) covers [-3, -3, -3, 3, 3, 3]. + - `center=0`, `base_scale=3`, `num_cascades=1` would create one grid (lvl=0) covers the aabb [[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]]. + - `center=0`, `base_scale=3`, `num_cascades=2` would create one grid (lvl=0) covers the aabb [[-1.5, -1.5, -1.5], [1.5, 1.5, 1.5]], + and one more grid (lvl=1) covers the aabb [[-3, -3, -3], [3, 3, 3]]. Normally you can simply set the `center` to the scene center, and `base_scale` to be the scene scale, and use `num_cascades=1` to create a single level density grid that covers the scene. @@ -68,11 +68,11 @@ class DensityGrid(nn.Module): TODO(ruilongli): support `center` and `base_scale` with 3-dim. Args: - center (float, optional): Center of all the grids in the world space. Defaults to 0. - base_scale (float, optional): Center of the scale of the base level grid. Defaults to 1. - num_cascades (int, optional): Number of the cascaded multi-res levels. Defaults to 1. - resolution (int, optional): Resolution of the grid. Defaults to 128. - update_every_num_iters (int, optional): How frequently to update the grid values. Defaults to 16. + center (float): Center of all the grids in the world space. Defaults to 0. + base_scale (float): Center of the scale of the base level grid. Defaults to 1. + num_cascades (int): Number of the cascaded multi-res levels. Defaults to 1. + resolution (int): Resolution of the grid. Defaults to 128. + update_every_num_iters (int): How frequently to update the grid values. Defaults to 16. """ def __init__( @@ -157,11 +157,11 @@ class DensityGrid(nn.Module): """Update the density grid in EMA way. Args: - density_eval_func (Callable): A Callable function that takes in sample positions (N, 3) and + density_eval_func: A Callable function that takes in sample positions (N, 3) and returns densities (N, 1). - step (int): The current training step. Instant-NGP has a warmup stage where all cells are updated. + step: The current training step. Instant-NGP has a warmup stage where all cells are updated. After certain amount of steps (256), it speeds up by only update sampled cells. - density_threshold (float): The threshold to prune cells. Recommand to calculate it using this rule: + density_threshold (float): The threshold to prune cells. Recommended to calculate it using this rule: `weight_threshold / dt`. For example for Instant-NGP on Lego, the minimum step size `dt` is `3` (scale of the scene) * `sqrt(3)` / `1024` (number of samples). And if we want to the weight threshold to be `0.01`, we will get `0.01 / (3 * sqrt(3) / 1024) ~= 2.` for density threshold. diff --git a/nerfactory/fields/instant_ngp_field.py b/nerfactory/fields/instant_ngp_field.py index 2ab94e7e4..834acd4bf 100644 --- a/nerfactory/fields/instant_ngp_field.py +++ b/nerfactory/fields/instant_ngp_field.py @@ -23,6 +23,7 @@ import torch from torch.nn.parameter import Parameter from nerfactory.cameras.rays import RaySamples +from nerfactory.data.structs import SceneBounds from nerfactory.fields.base import Field from nerfactory.fields.modules.encoding import Encoding, HashEncoding, SHEncoding from nerfactory.fields.modules.field_heads import FieldHeadNames @@ -36,13 +37,6 @@ except ImportError: pass -def get_normalized_positions(positions, aabb): - """Return normalized positions in range [0, 1] based on the aabb axis-aligned bounding box.""" - aabb_lengths = aabb[1] - aabb[0] - positions = (positions - aabb[0]) / aabb_lengths - return positions - - def get_normalized_directions(directions): """SH encoding must be in the range [0, 1]""" return (directions + 1.0) / 2.0 @@ -105,7 +99,7 @@ class TCNNInstantNGPField(Field): def get_density(self, ray_samples: RaySamples): """Computes and returns the densities.""" - positions = get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) + positions = SceneBounds.get_normalized_positions(ray_samples.frustums.get_positions(), self.aabb) positions_flat = positions.view(-1, 3) h = self.mlp_base(positions_flat).view(*ray_samples.frustums.shape, -1) density_before_activation, base_mlp_out = torch.split(h, [1, self.geo_feat_dim], dim=-1) @@ -156,7 +150,7 @@ class TorchInstantNGPField(NeRFField): def get_density(self, ray_samples: RaySamples): normalized_ray_samples = ray_samples - normalized_ray_samples.positions = get_normalized_positions( + normalized_ray_samples.positions = SceneBounds.get_normalized_positions( normalized_ray_samples.frustums.get_positions(), self.aabb ) return super().get_density(normalized_ray_samples) diff --git a/nerfactory/graphs/base.py b/nerfactory/graphs/base.py index 77cbe44eb..013c09306 100644 --- a/nerfactory/graphs/base.py +++ b/nerfactory/graphs/base.py @@ -30,6 +30,7 @@ from nerfactory.cameras.rays import RayBundle from nerfactory.data.structs import DatasetInputs, SceneBounds from nerfactory.graphs.modules.ray_generator import RayGenerator from nerfactory.utils import profiler +from nerfactory.utils.callbacks import Callback from nerfactory.utils.config import GraphConfig from nerfactory.utils.misc import ( get_masked_dict, @@ -58,10 +59,6 @@ class AbstractGraph(nn.Module): # to keep track of which device the nn.Module is on self.device_indicator_param = nn.Parameter(torch.empty(0)) - def get_device(self): - """Returns the device that the torch parameters are on.""" - return self.device_indicator_param.device - @property def device(self): """Returns the device that the graph is on.""" @@ -73,7 +70,8 @@ class AbstractGraph(nn.Module): class Graph(AbstractGraph): - """Where everything (Fields, Optimizers, Samplers, Visualization, etc) is linked together. This should be + """Graph class + Where everything (Fields, Optimizers, Samplers, Visualization, etc) is linked together. This should be subclassed for custom NeRF model. Args: @@ -122,9 +120,9 @@ class Graph(AbstractGraph): self.vis_outputs = None self.default_output_name = None - def register_callbacks(self): # pylint:disable=no-self-use - """Option to register callback for training functions""" - self.callbacks = [] + def get_training_callbacks(self) -> List[Callback]: # pylint:disable=no-self-use + """Returns a list of callbacks that run functions at the specified training iterations.""" + return [] def populate_density_field(self): """Set the scene density field to use.""" @@ -235,7 +233,7 @@ class Graph(AbstractGraph): def get_outputs_for_camera(self, camera: Camera): """Get the graph outputs for a Camera.""" - camera_ray_bundle = camera.get_camera_ray_bundle(device=self.get_device()) + camera_ray_bundle = camera.get_camera_ray_bundle(device=self.device) return self.get_outputs_for_camera_ray_bundle(camera_ray_bundle) @abstractmethod diff --git a/nerfactory/graphs/instant_ngp.py b/nerfactory/graphs/instant_ngp.py index 049b0ec00..b2103b008 100644 --- a/nerfactory/graphs/instant_ngp.py +++ b/nerfactory/graphs/instant_ngp.py @@ -36,22 +36,25 @@ from nerfactory.utils.callbacks import Callback class NGPGraph(Graph): + """Instant NGP graph - """Instant NGP graph""" + Args: + field_implementation (str): one of "torch" or "tcnn", or other fields in 'field_implementation_to_class' + kwargs: additional params to pass up to the parent class Graph + """ - def __init__(self, field_implementation="torch", intrinsics=None, camera_to_world=None, **kwargs) -> None: + def __init__(self, field_implementation="torch", **kwargs) -> None: assert field_implementation in field_implementation_to_class self.field_implementation = field_implementation self.field = None - super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs) + super().__init__(**kwargs) - def register_callbacks(self) -> None: - """defining callbacks to run after every training iteration""" + def get_training_callbacks(self) -> List[Callback]: assert self.density_field is not None - self.callbacks = [ + return [ Callback( - self.density_field.update_every_num_iters, - self.density_field.update_density_grid, + update_every_num_iters=self.density_field.update_every_num_iters, + func=self.density_field.update_density_grid, density_eval_func=self.field.density_fn, ) ] @@ -119,7 +122,7 @@ class NGPGraph(Graph): return outputs def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients): - device = self.get_device() + device = self.device image = batch["image"].to(device) if "alive_ray_mask" in outputs: mask = outputs["alive_ray_mask"] diff --git a/nerfactory/graphs/vanilla_nerf.py b/nerfactory/graphs/vanilla_nerf.py index 5604f731a..3e2e687fd 100644 --- a/nerfactory/graphs/vanilla_nerf.py +++ b/nerfactory/graphs/vanilla_nerf.py @@ -82,17 +82,17 @@ class NeRFGraph(Graph): **kwargs, ) - def register_callbacks(self) -> None: - """defining callbacks to run after every training iteration""" - self.callbacks = [] + def get_training_callbacks(self) -> List[Callback]: + callbacks = [] if self.density_field is not None: - self.callbacks += [ + callbacks = [ Callback( - self.density_field.update_every_num_iters, - self.density_field.update_density_grid, + update_every_num_iters=self.density_field.update_every_num_iters, + func=self.density_field.update_density_grid, density_eval_func=self.field_coarse.density_fn, ) ] + return callbacks # type: ignore def populate_fields(self): """Set the fields.""" diff --git a/nerfactory/utils/callbacks.py b/nerfactory/utils/callbacks.py index c2de09f49..f48d8d84b 100644 --- a/nerfactory/utils/callbacks.py +++ b/nerfactory/utils/callbacks.py @@ -13,16 +13,26 @@ # limitations under the License. """ -Callback functions for training iterations +Callback code used for training iterations """ from typing import Callable class Callback: - """Callback Object""" + """Callback class used during training. + The function 'func' with 'args' and 'kwargs' will be called every 'update_every_num_iters' training iterations, + including at iteration 0. + + Args: + update_every_num_iters: How often to call the function `func`. + func: The function that will be called. + args: args for the function 'func'. + kwargs: kwargs for the function 'func'. + """ def __init__(self, update_every_num_iters: int, func: Callable, *args, **kwargs): + # TODO(ethan): how do we type args and kwargs? self.update_every_num_iters = update_every_num_iters self.func = func self.args = args diff --git a/nerfactory/utils/plotly.py b/nerfactory/utils/plotly.py index b75321bc6..5722b0b61 100644 --- a/nerfactory/utils/plotly.py +++ b/nerfactory/utils/plotly.py @@ -14,6 +14,11 @@ """ Visualization code for plotly. +The function use prefix conventions in the following way: + - 'get_*' functions (e.g., 'get_camera_frustums') + return data that can be ploted with plotly + - 'vis_*' functions (e.g., 'vis_camera_rays') + return 'go.Figure' objects which are the plots. Go Figure! :') """ from typing import List, Union @@ -93,7 +98,7 @@ def get_line_segments_from_lines( return data -def visualize_dataset(camera_origins: TensorType["num_cameras", 3], ray_bundle: RayBundle) -> go.FigureWidget: +def vis_dataset(camera_origins: TensorType["num_cameras", 3], ray_bundle: RayBundle) -> go.FigureWidget: """Visualize a dataset with plotly using our cameras and generated rays. Args: @@ -490,3 +495,18 @@ def vis_camera_rays(camera: Camera) -> go.Figure: ) return fig + + +def get_camera_frustums(cameras: List[Camera]): + """Returns the camera frustums for the cameras that we are using. + + Args: + cameras: The cameras that we want to plot. + + Returns: + A plotly scatter that can be plotted. + """ + for camera in cameras: + json_ = camera.to_json() + print(json_) + raise NotImplementedError diff --git a/nerfactory/viewer/server/viewer_utils.py b/nerfactory/viewer/server/viewer_utils.py index 4816272e6..1cd02c356 100644 --- a/nerfactory/viewer/server/viewer_utils.py +++ b/nerfactory/viewer/server/viewer_utils.py @@ -342,7 +342,7 @@ class VisualizerState: ) intrinsics = get_intrinsics_from_intrinsics_matrix(intrinsics_matrix) camera = get_camera(intrinsics, camera_to_world) - camera_ray_bundle = camera.get_camera_ray_bundle(device=graph.get_device()) + camera_ray_bundle = camera.get_camera_ray_bundle(device=graph.device) camera_ray_bundle.num_rays_per_chunk = self.config.num_rays_per_chunk graph.eval()