From fb440fedf37f78b8301281dffb432be7394b2608 Mon Sep 17 00:00:00 2001 From: Ethan Weber Date: Tue, 17 May 2022 19:09:25 -0700 Subject: [PATCH] Occupancy grid (#79) * occupancy grid in progress * visualizing occupancy grid. currently restructing field and renderer head outputs * black and pylint. broke nerfw and semantic nerf but it's okay * get_project_root --- README.md | 6 +- environment/run_setup_environment.sh | 1 - .../scene_bounds_collider.py => colliders.py} | 0 mattport/nerf/dataset/format/blender.py | 3 +- mattport/nerf/field_modules/field_heads.py | 28 ++-- mattport/nerf/fields/__init__.py | 0 mattport/nerf/fields/base.py | 51 +++++++ mattport/nerf/graph/base.py | 135 ++++++++--------- mattport/nerf/graph/vanilla_nerf.py | 139 ++++++++++-------- mattport/nerf/occupancy_grid.py | 119 +++++++++++++++ .../nerf/{field_modules => }/ray_generator.py | 0 mattport/nerf/renderers.py | 57 +++---- mattport/nerf/trainer.py | 2 +- mattport/structures/rays.py | 14 ++ mattport/utils/io.py | 15 +- mattport/utils/stats_tracker.py | 14 +- pyproject.toml | 2 +- setup.py | 2 +- tests/nerf/dataloader/test_ray_generator.py | 2 +- .../nerf/field_modules/test_field_outputs.py | 4 +- tests/nerf/test_occupancy_grid.py | 22 +++ tests/nerf/test_renderers.py | 22 +-- tests/nerf/test_sampler.py | 2 +- 23 files changed, 423 insertions(+), 217 deletions(-) delete mode 100644 environment/run_setup_environment.sh rename mattport/nerf/{field_modules/scene_bounds_collider.py => colliders.py} (100%) create mode 100644 mattport/nerf/fields/__init__.py create mode 100644 mattport/nerf/fields/base.py create mode 100644 mattport/nerf/occupancy_grid.py rename mattport/nerf/{field_modules => }/ray_generator.py (100%) create mode 100644 tests/nerf/test_occupancy_grid.py diff --git a/README.md b/README.md index c96306876..20ffddb2b 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,9 @@ pip install -r environment/requirements.txt # Install mattport as a library pip install -e . +# Install library with CUDA support. Change setup.py to `USE_CUDA = True` and then +python setup.py develop + # Running the test cases pytest tests ``` @@ -114,6 +117,7 @@ ipython --pdb scripts/run_train.py For testing specific methods, see scripts/benchmarking/run_benchmark.py. Modify the `BENCH` variable to specify which jobs ("ckpt_dir") and methods ("method") you want benchmarked. Then run + ``` python scripts/benchmarking/run_benchmark.py -``` \ No newline at end of file +``` diff --git a/environment/run_setup_environment.sh b/environment/run_setup_environment.sh deleted file mode 100644 index 29851782a..000000000 --- a/environment/run_setup_environment.sh +++ /dev/null @@ -1 +0,0 @@ -PROJECT_ROOT=`pwd` diff --git a/mattport/nerf/field_modules/scene_bounds_collider.py b/mattport/nerf/colliders.py similarity index 100% rename from mattport/nerf/field_modules/scene_bounds_collider.py rename to mattport/nerf/colliders.py diff --git a/mattport/nerf/dataset/format/blender.py b/mattport/nerf/dataset/format/blender.py index d66ac29f0..2ffa1e20a 100644 --- a/mattport/nerf/dataset/format/blender.py +++ b/mattport/nerf/dataset/format/blender.py @@ -53,7 +53,8 @@ def load_blender_data( intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32) intrinsics *= torch.tensor([cx, cy, focal_length]) - scene_bounds = SceneBounds(aabb=torch.tensor([[-1, -1, -1], [1, 1, 1]], dtype=torch.float32)) + # in x,y,z order + scene_bounds = SceneBounds(aabb=torch.tensor([[-2, -2, -2], [2, 2, 2]], dtype=torch.float32)) dataset_inputs = DatasetInputs( image_filenames=image_filenames, diff --git a/mattport/nerf/field_modules/field_heads.py b/mattport/nerf/field_modules/field_heads.py index 3d61d22d0..4416843b4 100644 --- a/mattport/nerf/field_modules/field_heads.py +++ b/mattport/nerf/field_modules/field_heads.py @@ -2,7 +2,7 @@ Collection of render heads """ from enum import Enum -from typing import Dict, Optional +from typing import Optional from torch import nn from torchtyping import TensorType @@ -26,27 +26,23 @@ class FieldHeadNames(Enum): class FieldHead(FieldModule): """Base field output""" - def __init__( - self, in_dim: int, out_dim: int, field_head_name: FieldHeadNames, activation: Optional[nn.Module] = None - ) -> None: + def __init__(self, in_dim: int, out_dim: int, activation: Optional[nn.Module] = None) -> None: """ Args: in_dim (int): input dimension out_dim (int): output dimension for renderer - field_head_name (FieldHeadNames): name of field head activation (Optional[nn.Module]): output head activation """ super().__init__() self.in_dim = in_dim self.out_dim = out_dim - self.field_head_name = field_head_name self.activation = activation layers = [nn.Linear(self.in_dim, self.out_dim)] if self.activation: layers.append(self.activation) self.net = nn.Sequential(*layers) - def forward(self, in_tensor: TensorType[..., "in_dim"]) -> Dict[FieldHeadNames, TensorType]: + def forward(self, in_tensor: TensorType[..., "in_dim"]) -> TensorType: """Process network output for renderer Args: @@ -55,22 +51,24 @@ class FieldHead(FieldModule): Returns: TensorType[..., "out_dim"]: Render head output """ + if not self.net: + raise SystemError("Render head network not initialized. build_nn_modules() should be called.") out_tensor = self.net(in_tensor) - return {self.field_head_name: out_tensor} + return out_tensor class DensityFieldHead(FieldHead): """Density output""" def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Softplus()) -> None: - super().__init__(in_dim, out_dim=1, field_head_name=FieldHeadNames.DENSITY, activation=activation) + super().__init__(in_dim, out_dim=1, activation=activation) class RGBFieldHead(FieldHead): """RGB output""" def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Sigmoid()) -> None: - super().__init__(in_dim, out_dim=3, field_head_name=FieldHeadNames.RGB, activation=activation) + super().__init__(in_dim, out_dim=3, activation=activation) class SHFieldHead(FieldHead): @@ -86,28 +84,28 @@ class SHFieldHead(FieldHead): activation (Optional[nn.Module], optional): Output activation. Defaults to None. """ out_dim = channels * levels**2 - super().__init__(in_dim, out_dim=out_dim, field_head_name=FieldHeadNames.SH, activation=activation) + super().__init__(in_dim, out_dim=out_dim, activation=activation) class UncertaintyFieldHead(FieldHead): """Uncertainty output""" def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Softplus()) -> None: - super().__init__(in_dim, out_dim=1, field_head_name=FieldHeadNames.UNCERTAINTY, activation=activation) + super().__init__(in_dim, out_dim=1, activation=activation) class TransientRGBHead(FieldHead): """Transient RGB output""" def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Sigmoid()) -> None: - super().__init__(in_dim, out_dim=3, field_head_name=FieldHeadNames.TRANSIENT_RGB, activation=activation) + super().__init__(in_dim, out_dim=3, activation=activation) class TransientDensityHead(FieldHead): """Transient density output""" def __init__(self, in_dim: int, activation: Optional[nn.Module] = nn.Softplus()) -> None: - super().__init__(in_dim, out_dim=1, field_head_name=FieldHeadNames.TRANSIENT_DENSITY, activation=activation) + super().__init__(in_dim, out_dim=1, activation=activation) class SemanticStuffHead(FieldHead): @@ -119,4 +117,4 @@ class SemanticStuffHead(FieldHead): in_dim (int): Input dimension num_classes (int): Number of semantic classes """ - super().__init__(in_dim, out_dim=num_classes, field_head_name=FieldHeadNames.SEMANTICS_STUFF, activation=None) + super().__init__(in_dim, out_dim=num_classes, activation=None) diff --git a/mattport/nerf/fields/__init__.py b/mattport/nerf/fields/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mattport/nerf/fields/base.py b/mattport/nerf/fields/base.py new file mode 100644 index 000000000..5f3b01f37 --- /dev/null +++ b/mattport/nerf/fields/base.py @@ -0,0 +1,51 @@ +""" +Base class for the graphs. +""" + +from abc import abstractmethod + +from torch import nn + +from mattport.structures.rays import PointSamples, RaySamples + + +class Field(nn.Module): + """Base class for fields.""" + + def __init__(self) -> None: + super().__init__() + # minimum weight for it to be valid. used to skip querying the field + # TODO(ethan): avoid the hardcoding + self.weights_threshold = 1e-4 + + def density_fn(self, positions): + """Returns only the density. Used primarily with the occupancy grid.""" + point_samples = PointSamples(positions=positions) + density, _ = self.get_density(point_samples) + return density + + @abstractmethod + def get_density(self, point_samples: PointSamples, valid_mask=None): + """Computes and returns the densities.""" + + @abstractmethod + def get_outputs(self, point_samples: PointSamples, density_embedding=None, valid_mask=None): + """Computes and returns the colors.""" + + def forward(self, point_samples: PointSamples, valid_mask_density=None): + """Evaluates the field at points along the ray.""" + density, density_embedding = self.get_density(point_samples, valid_mask=valid_mask_density) + field_outputs = self.get_outputs(point_samples, density_embedding=density_embedding) + field_outputs["density"] = density + return field_outputs + + def forward_with_weight_pruning(self, ray_samples: RaySamples, valid_mask_density=None): + """Forward but with pruning based on weights, from the densities.""" + density, density_embedding = self.get_density(ray_samples, valid_mask=valid_mask_density) + weights = ray_samples.get_weights(density) + valid_mask_outputs = weights >= self.weights_threshold + field_outputs = self.get_outputs( + ray_samples, density_embedding=density_embedding, valid_mask=valid_mask_outputs + ) + field_outputs["density"] = density + return field_outputs diff --git a/mattport/nerf/graph/base.py b/mattport/nerf/graph/base.py index 8f52b133e..f10c88213 100644 --- a/mattport/nerf/graph/base.py +++ b/mattport/nerf/graph/base.py @@ -1,10 +1,10 @@ """ The Graph module contains all trainable parameters. """ +import logging from abc import abstractmethod from collections import defaultdict -from dataclasses import dataclass -from typing import Dict, List, Optional, Union +from typing import Dict, List, Union import torch from omegaconf import DictConfig @@ -13,57 +13,20 @@ from torch.nn import Parameter from torchtyping import TensorType from mattport.nerf.dataset.structs import SceneBounds -from mattport.nerf.field_modules.ray_generator import RayGenerator -from mattport.nerf.field_modules.scene_bounds_collider import AABBBoxCollider +from mattport.nerf.occupancy_grid import OccupancyGrid +from mattport.nerf.ray_generator import RayGenerator +from mattport.nerf.colliders import AABBBoxCollider from mattport.structures.cameras import get_camera_model from mattport.structures.rays import RayBundle from mattport.utils.misc import get_masked_dict +from mattport.utils.misc import is_not_none -@dataclass -class GraphInputs: - """Datastucture to encode the inputs to the graph.""" +class AbstractGraph(nn.Module): + """Highest level graph class. Somewhat useful to lift code up and out of the way.""" - points: float - - -@dataclass -class Node: - """Node datastructure for graph composition.""" - - name: str - children: Dict[str, "Node"] - parents: Dict[str, "Node"] - visited_order: Optional[bool] = False - visited_in_dim: Optional[bool] = False - - def __hash__(self): - return hash(self.name) - - -class Graph(nn.Module): - """_summary_""" - - def __init__( - self, - intrinsics=None, - camera_to_world=None, - loss_coefficients: DictConfig = None, - scene_bounds: SceneBounds = None, - **kwargs, - ) -> None: + def __init__(self) -> None: super().__init__() - self.intrinsics = intrinsics - self.camera_to_world = camera_to_world - self.scene_bounds = scene_bounds - self.loss_coefficients = loss_coefficients - # assert self.scene_bounds is not None - self.kwargs = kwargs - self.scene_bounds_collider = None - self.ray_generator = RayGenerator(self.intrinsics, self.camera_to_world) - self.populate_collider() - self.populate_modules() # populate the modules - # to keep track of which device the nn.Module is on self.device_indicator_param = nn.Parameter(torch.empty(0)) @@ -71,45 +34,77 @@ class Graph(nn.Module): """Returns the device that the torch parameters are on.""" return self.device_indicator_param.device + @abstractmethod + def forward( + self, + ray_indices: TensorType["num_rays", 3], + batch: Union[str, Dict[str, torch.tensor]] = None, + step: int = None, + ): + """Process starting with ray indices. Turns them into rays, then performs volume rendering.""" + + +class Graph(AbstractGraph): + """_summary_""" + + def __init__( + self, + intrinsics=None, + camera_to_world=None, + loss_coefficients: DictConfig = None, + steps_per_occupancy_grid_update=16, + scene_bounds: SceneBounds = None, + **kwargs, + ) -> None: + super().__init__() + assert is_not_none(scene_bounds), "scene_bounds is needed to use the occupancy grid" + self.intrinsics = intrinsics + self.camera_to_world = camera_to_world + self.scene_bounds = scene_bounds + self.loss_coefficients = loss_coefficients + self.steps_per_occupancy_grid_update = steps_per_occupancy_grid_update + self.kwargs = kwargs + self.collider = None + self.occupancy_grid = OccupancyGrid(aabb=self.scene_bounds.aabb) + self.ray_generator = RayGenerator(self.intrinsics, self.camera_to_world) + self.populate_collider() + self.populate_fields() + self.populate_misc_modules() # populate the modules + def populate_collider(self): """Set the scene bounds collider to use.""" - self.scene_bounds_collider = AABBBoxCollider(self.scene_bounds) + self.collider = AABBBoxCollider(self.scene_bounds) @abstractmethod - def populate_modules(self): - """Initializes the modules that are part of the network.""" + def populate_misc_modules(self): + """Initializes any additional modules that are part of the network.""" @abstractmethod def get_param_groups(self) -> Dict[str, List[Parameter]]: - """Returns the parameter groups of the network in a dictionary format""" + """Obtain the parameter groups for the optimizers - def get_in_dim(self, curr_node: Node) -> None: - """Dynamically calculates and sets the input dimensions of the modules based on dependency graph - - Args: - curr_node (Node): pointer to current node in process + Returns: + Dict[str, List[Parameter]]: Mapping of different parameter groups """ - curr_node.visited_in_dim = True - if len(curr_node.parents) > 0: - in_dim = 0 - for parent_name in curr_node.parents.keys(): - in_dim += self[parent_name].get_out_dim() - self[curr_node.name].set_in_dim(in_dim) - self.modules[curr_node.name].meta_data.in_dim = in_dim - - for child_node in curr_node.children.values(): - if not child_node.visited_in_dim: - self.get_in_dim(child_node) @abstractmethod def get_outputs(self, ray_bundle: RayBundle): """Takes in a Ray Bundle and returns a dictionary of outputs.""" - def forward(self, ray_indices: TensorType["num_rays", 3], batch: Union[str, Dict[str, torch.tensor]] = None): - """Forward function that needs to be overridden.""" + def forward( + self, + ray_indices: TensorType["num_rays", 3], + batch: Union[str, Dict[str, torch.tensor]] = None, + step: int = None, + ): + # update the occupancy grid + if step and step % self.steps_per_occupancy_grid_update == 0: + logging.info("Updating occupancy grid.") + self.occupancy_grid.update_occupancy_grid(density_fn=self.field_fine.density_fn) + # get the rays original_ray_bundle = self.ray_generator.forward(ray_indices) # RayBundle - intersected_ray_bundle = self.scene_bounds_collider(original_ray_bundle) + intersected_ray_bundle = self.collider(original_ray_bundle) if isinstance(batch, type(None)): # during inference, keep all rays @@ -159,7 +154,7 @@ class Graph(nn.Module): end_idx = i + chunk_size original_ray_bundle = camera_ray_bundle.get_row_major_sliced_ray_bundle(start_idx, end_idx) original_ray_bundle.move_to_device(device) - intersected_ray_bundle = self.scene_bounds_collider(original_ray_bundle) + intersected_ray_bundle = self.collider(original_ray_bundle) outputs = self.get_outputs(intersected_ray_bundle) for output_name, output in outputs.items(): outputs_lists[output_name].append(output) diff --git a/mattport/nerf/graph/vanilla_nerf.py b/mattport/nerf/graph/vanilla_nerf.py index 931995969..e548ccbe5 100644 --- a/mattport/nerf/graph/vanilla_nerf.py +++ b/mattport/nerf/graph/vanilla_nerf.py @@ -12,21 +12,25 @@ from torchmetrics import PeakSignalNoiseRatio, StructuralSimilarityIndexMeasure from torchmetrics.image.lpip import LearnedPerceptualImagePatchSimilarity from mattport.nerf.field_modules.encoding import NeRFEncoding -from mattport.nerf.field_modules.field_heads import DensityFieldHead, FieldHeadNames, RGBFieldHead +from mattport.nerf.field_modules.field_heads import DensityFieldHead, RGBFieldHead from mattport.nerf.field_modules.mlp import MLP +from mattport.nerf.fields.base import Field from mattport.nerf.graph.base import Graph from mattport.nerf.loss import MSELoss from mattport.nerf.renderers import AccumulationRenderer, DepthRenderer, RGBRenderer from mattport.nerf.sampler import PDFSampler, UniformSampler from mattport.structures import colors -from mattport.structures.rays import RayBundle, RaySamples -from mattport.nerf.field_modules.scene_bounds_collider import NearFarCollider +from mattport.structures.rays import PointSamples, RayBundle +from mattport.nerf.colliders import NearFarCollider from mattport.utils import stats_tracker, visualization, writer +from mattport.utils.misc import is_not_none -class NeRFField(nn.Module): +class NeRFField(Field): """NeRF module""" + OUTPUT_RGB = "rgb" + def __init__(self, num_layers=8, layer_width=256, skip_connections: Tuple = (4,)) -> None: super().__init__() self.num_layers = num_layers @@ -73,22 +77,35 @@ class NeRFField(nn.Module): self.field_output_rgb = RGBFieldHead(in_dim=self.mlp_rgb.get_out_dim()) self.field_output_density = DensityFieldHead(in_dim=self.mlp_base.get_out_dim()) - def forward(self, ray_samples: RaySamples): - """Evaluates the field at points along the ray.""" - positions = ray_samples.positions - directions = ray_samples.directions - encoded_xyz = self.encoding_xyz(positions) - encoded_dir = self.encoding_dir(directions) - base_mlp_out = self.mlp_base(encoded_xyz) - rgb_mlp_out = self.mlp_rgb(torch.cat([encoded_dir, base_mlp_out], dim=-1)) + def get_density(self, point_samples: PointSamples, valid_mask=None): + """Computes and returns the densities.""" + positions = point_samples.positions + if not is_not_none(valid_mask): + valid_mask = torch.ones_like(positions[..., 0]).bool() + # placeholders for values to return + density = torch.zeros(*valid_mask.shape, 1, dtype=torch.float32, device=positions.device) + base_mlp_out = torch.zeros( + *valid_mask.shape, self.mlp_base.out_dim, dtype=torch.float32, device=positions.device + ) + if not valid_mask.any(): # empty mask + return density, base_mlp_out + encoded_xyz = self.encoding_xyz(positions[valid_mask]) + base_mlp_out[valid_mask] = self.mlp_base(encoded_xyz) + density[valid_mask] = self.field_output_density(base_mlp_out[valid_mask]) + return density, base_mlp_out - field_rgb_output = self.field_output_rgb(rgb_mlp_out) - field_density_out = self.field_output_density(base_mlp_out) - - field_outputs = {} - field_outputs.update(field_rgb_output) - field_outputs.update(field_density_out) - return field_outputs + def get_outputs(self, point_samples: PointSamples, density_embedding=None, valid_mask=None): + directions = point_samples.directions + if not is_not_none(valid_mask): + valid_mask = torch.ones_like(directions[..., 0]).bool() + # placeholders for values to return + rgb = torch.zeros(*valid_mask.shape, 3, dtype=torch.float32, device=directions.device) + if not valid_mask.any(): # empty mask + return {"rgb": rgb} + encoded_dir = self.encoding_dir(directions[valid_mask]) + rgb_mlp_out = self.mlp_rgb(torch.cat([encoded_dir, density_embedding[valid_mask]], dim=-1)) + rgb[valid_mask] = self.field_output_rgb(rgb_mlp_out) + return {"rgb": rgb} class NeRFGraph(Graph): @@ -112,23 +129,20 @@ class NeRFGraph(Graph): self.field_fine = None super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world, **kwargs) + def populate_collider(self): + self.collider = NearFarCollider(self.near_plane, self.far_plane) + def populate_fields(self): """Set the fields.""" self.field_coarse = NeRFField() self.field_fine = NeRFField() - def populate_collider(self): - self.scene_bounds_collider = NearFarCollider(self.near_plane, self.far_plane) - - def populate_modules(self): + def populate_misc_modules(self): # samplers self.sampler_uniform = UniformSampler(num_samples=self.num_coarse_samples) self.sampler_pdf = PDFSampler(num_samples=self.num_importance_samples) - # field - self.populate_fields() - # renderers self.renderer_rgb = RGBRenderer(background_color=colors.WHITE) self.renderer_accumulation = AccumulationRenderer() @@ -143,54 +157,52 @@ class NeRFGraph(Graph): self.lpips = LearnedPerceptualImagePatchSimilarity() def get_param_groups(self) -> Dict[str, List[Parameter]]: - """Obtain the parameter groups for the optimizers - - Returns: - Dict[str, List[Parameter]]: Mapping of different parameter groups - """ param_groups = {} param_groups["fields"] = list(self.field_coarse.parameters()) + list(self.field_fine.parameters()) return param_groups def get_outputs(self, ray_bundle: RayBundle): - # TODO: set the new rays... + # uniform sampling + ray_samples_uniform = self.sampler_uniform(ray_bundle) - # coarse network: - uniform_ray_samples = self.sampler_uniform(ray_bundle) # RaySamples + # coarse field: + field_outputs_coarse = self.field_coarse.forward(ray_samples_uniform.to_point_samples()) + weights_coarse = ray_samples_uniform.get_weights(field_outputs_coarse["density"]) + rgb_coarse = self.renderer_rgb( + rgb=field_outputs_coarse["rgb"], + weights=weights_coarse, + ) + accumulation_coarse = self.renderer_accumulation(weights_coarse) + depth_coarse = self.renderer_depth(weights_coarse, ray_samples_uniform.ts) - coarse_field_outputs = self.field_coarse(uniform_ray_samples) # FieldOutputs + # pdf sampling + ray_samples_pdf = self.sampler_pdf(ray_samples_uniform, weights_coarse) - coarse_weights = uniform_ray_samples.get_weights(coarse_field_outputs[FieldHeadNames.DENSITY]) + # fine field: + field_outputs_fine = self.field_fine.forward(ray_samples_pdf.to_point_samples()) + weights_fine = ray_samples_pdf.get_weights(field_outputs_fine["density"]) + rgb_fine = self.renderer_rgb( + rgb=field_outputs_fine["rgb"], + weights=weights_fine, + ) + accumulation_fine = self.renderer_accumulation(weights_fine) + depth_fine = self.renderer_depth(weights_fine, ray_samples_pdf.ts) - coarse_renderer_outputs = self.renderer_rgb( - rgb=coarse_field_outputs[FieldHeadNames.RGB], - weights=coarse_weights, - ) # RendererOutputs - coarse_renderer_accumulation = self.renderer_accumulation(coarse_weights) # RendererOutputs - coarse_renderer_depth = self.renderer_depth(coarse_weights, uniform_ray_samples.ts) + # --- occupancy grid --- + # TODO(ethan): move this elsewhere. currently here for debugging purposes + density_occ = self.occupancy_grid.get_densities(ray_samples_pdf.positions) + weights_occ = ray_samples_pdf.get_weights(density_occ) + depth_occ = self.renderer_depth(weights_occ, ray_samples_pdf.ts) - # fine network: - pdf_ray_samples = self.sampler_pdf(uniform_ray_samples, coarse_weights) # RaySamples - fine_field_outputs = self.field_fine(pdf_ray_samples) # FieldOutputs - - fine_weights = pdf_ray_samples.get_weights(fine_field_outputs[FieldHeadNames.DENSITY]) - - fine_renderer_outputs = self.renderer_rgb( - rgb=fine_field_outputs[FieldHeadNames.RGB], - weights=fine_weights, - ) # RendererOutputs - fine_renderer_accumulation = self.renderer_accumulation(fine_weights) # RendererOutputs - fine_renderer_depth = self.renderer_depth(fine_weights, pdf_ray_samples.ts) - - # outputs: outputs = { - "rgb_coarse": coarse_renderer_outputs.rgb, - "rgb_fine": fine_renderer_outputs.rgb, - "accumulation_coarse": coarse_renderer_accumulation.accumulation, - "accumulation_fine": fine_renderer_accumulation.accumulation, - "depth_coarse": coarse_renderer_depth.depth, - "depth_fine": fine_renderer_depth.depth, + "rgb_coarse": rgb_coarse, + "rgb_fine": rgb_fine, + "accumulation_coarse": accumulation_coarse, + "accumulation_fine": accumulation_fine, + "depth_coarse": depth_coarse, + "depth_fine": depth_fine, + "depth_occ": depth_occ, } return outputs @@ -229,6 +241,9 @@ class NeRFGraph(Graph): writer.write_image(name=f"image_idx_{image_idx}", image=combined_acc, step=step, group="accumulation") writer.write_image(name=f"image_idx_{image_idx}", image=combined_depth, step=step, group="depth") + depth_occ = visualization.apply_depth_colormap(outputs["depth_occ"]) + writer.write_image(name=f"image_idx_{image_idx}", image=depth_occ, step=step, group="depth_occupancy_grid") + # Switch images from [H, W, C] to [1, C, H, W] for metrics computations image = torch.moveaxis(image, -1, 0)[None, ...] rgb_coarse = torch.moveaxis(rgb_coarse, -1, 0)[None, ...] diff --git a/mattport/nerf/occupancy_grid.py b/mattport/nerf/occupancy_grid.py new file mode 100644 index 000000000..f44679c6a --- /dev/null +++ b/mattport/nerf/occupancy_grid.py @@ -0,0 +1,119 @@ +""" +Code to implement the density grid. +""" + +import torch +import torch.nn.functional as F +from torch import nn +from torchtyping import TensorType + +from mattport.utils.misc import is_not_none + + +class OccupancyGrid(nn.Module): + """Module to keep track of the density and occupancy.""" + + def __init__(self, num_cascades=1, resolution=128, aabb=None) -> None: + super().__init__() + assert is_not_none(aabb), "The axis-aligned bounding box aabb is not defined!" + self.num_cascades = num_cascades # the number of levels (i.e, cascades) + self.resolution = resolution + self.register_buffer("aabb", aabb) # axis-aligned bounding box + occupancy_grid = torch.zeros( + [self.num_cascades] + [self.resolution] * 3 + ) # (num_cascades, x_resolution, y_resolution, z_resolution) + # not a module parameter, but it still should be saved and part of `state_dict` + self.register_buffer("occupancy_grid", occupancy_grid) + self.mean_density = 0.0 + + def reset(self): + """Zero out the occupancy grid.""" + self.occupancy_grid.zero_() + + @torch.no_grad() + def mark_untrained_grid(self): + """Marks the locations in the density grid that are never seen by training views.""" + # TODO: only query points that are covered by the training cameras viewing frustrum + # We'll need to pass in the camera poses and intrinsics to this function + raise NotImplementedError() + + @torch.no_grad() + def update_occupancy_grid(self, density_fn, decay=0.95, split_size=64): + """Update the occupancy grid with density values. + + Args: + density_fn: Function from takes points in the shape (..., 3) and + returns a tensor of densities of shape (..., 1). + """ + # TODO(ethan): remove this pylint disable + # pylint: disable=invalid-name + + device = self.occupancy_grid.device + + assert self.num_cascades == 1, "Still need to implement code for multiple cascades." + temp_occupancy_grid = torch.zeros_like(self.occupancy_grid) + + X_length, Y_length, Z_length = self.aabb[1] - self.aabb[0] + X_half_voxel_length = 0.5 * X_length / self.resolution + Y_half_voxel_length = 0.5 * Y_length / self.resolution + Z_half_voxel_length = 0.5 * Z_length / self.resolution + half_voxel_lengths = torch.tensor( + [X_half_voxel_length, Y_half_voxel_length, Z_half_voxel_length], device=device + ) + X = torch.linspace( + self.aabb[0][0] + X_half_voxel_length, self.aabb[1][0] - X_half_voxel_length, self.resolution, device=device + ).split(split_size) + Y = torch.linspace( + self.aabb[0][1] + Y_half_voxel_length, self.aabb[1][1] - Y_half_voxel_length, self.resolution, device=device + ).split(split_size) + Z = torch.linspace( + self.aabb[0][2] + Z_half_voxel_length, self.aabb[1][2] - Z_half_voxel_length, self.resolution, device=device + ).split(split_size) + for x_index, xs in enumerate(X): + for y_index, ys in enumerate(Y): + for z_index, zs in enumerate(Z): + x_len, y_len, z_len = len(xs), len(ys), len(zs) + grid_x, grid_y, grid_z = torch.meshgrid(xs, ys, zs, indexing="ij") + xyzs = torch.stack([grid_x, grid_y, grid_z], dim=-1) # (resolution, resolution,, resolution, 3) + # add noise to the points + xyzs_rand = ( + torch.rand_like(xyzs) * half_voxel_lengths + ) # range [-half_voxel_length, half_voxel_length] for each dimension + xyzs += xyzs_rand + densities = density_fn(xyzs) + assert densities.shape[-1] == 1 + temp_occupancy_grid[ + 0, + x_index * split_size : x_index * split_size + x_len, + y_index * split_size : y_index * split_size + y_len, + z_index * split_size : z_index * split_size + z_len, + ] = densities[..., 0] + + # exponential moving average update + valid_mask = self.occupancy_grid >= 0.0 + self.occupancy_grid[valid_mask] = torch.maximum( + self.occupancy_grid[valid_mask] * decay, temp_occupancy_grid[valid_mask] + ) + self.mean_density = torch.mean(self.occupancy_grid[valid_mask]).item() + + def get_densities(self, xyzs: TensorType[..., 3]): + """Trilinear interpolation to get the density values.""" + occupancy_grid = self.occupancy_grid[None, ...] # shape (1, num_cascades, X_res, Y_res, Z_res) + xyzs_shape = xyzs.shape + xyzs_reshaped = xyzs.view(1, -1, 1, 1, 3) + voxel_lengths = self.aabb[1] - self.aabb[0] + xyzs_reshaped_normalized = ((xyzs_reshaped - self.aabb[0]) / voxel_lengths) * 2.0 - 1.0 + densities = F.grid_sample( + occupancy_grid.permute( + 0, 1, 4, 3, 2 + ), # (xyz to zyx) for grid sample useage because the xyzs have xyz ordering + xyzs_reshaped_normalized, + align_corners=True, + padding_mode="zeros", + ) + densities = densities.view(*xyzs_shape[:-1], 1) + return densities + + def forward(self, x): + """Not implemented.""" + raise NotImplementedError() diff --git a/mattport/nerf/field_modules/ray_generator.py b/mattport/nerf/ray_generator.py similarity index 100% rename from mattport/nerf/field_modules/ray_generator.py rename to mattport/nerf/ray_generator.py diff --git a/mattport/nerf/renderers.py b/mattport/nerf/renderers.py index eb9174b71..86843ee5c 100644 --- a/mattport/nerf/renderers.py +++ b/mattport/nerf/renderers.py @@ -1,7 +1,6 @@ """ Collection of renderers """ -from dataclasses import dataclass import math from typing import Optional @@ -12,18 +11,6 @@ from torchtyping import TensorType from mattport.utils.math import components_from_spherical_harmonics -@dataclass -class RendererOutputs: - """_summary_""" - - rgb: TensorType["num_rays", 3] = None - density: TensorType["num_rays", 1] = None - accumulation: TensorType["num_rays", 1] = None - depth: TensorType["num_rays", 1] = None - uncertainty: TensorType["num_rays", 1] = None - semantics: TensorType["num_rays", "num_classes"] = None - - class RGBRenderer(nn.Module): """Standard volumetic rendering.""" @@ -62,7 +49,7 @@ class RGBRenderer(nn.Module): self, rgb: TensorType[..., "num_samples", 3], weights: TensorType[..., "num_samples"], - ) -> RendererOutputs: + ) -> TensorType[..., 3]: """Composite samples along ray and render color image Args: @@ -70,14 +57,13 @@ class RGBRenderer(nn.Module): weights (TensorType[..., "num_samples"]): Weights for each sample Returns: - RendererOutputs: Outputs with rgb values. + TensorType[..., 3]: Outputs of rgb values. """ rgb = self.combine_rgb(rgb, weights) if not self.training: torch.clamp_(rgb, min=0.0, max=1.0) - renderer_outputs = RendererOutputs(rgb=rgb) - return renderer_outputs + return rgb class SHRenderer(nn.Module): @@ -100,7 +86,7 @@ class SHRenderer(nn.Module): sh: TensorType[..., "num_samples", "coeffs"], directions: TensorType[..., "num_samples", 3], weights: TensorType[..., "num_samples"], - ) -> RendererOutputs: + ) -> TensorType[..., 3]: """Composite samples along ray and render color image Args: @@ -109,7 +95,7 @@ class SHRenderer(nn.Module): weights (TensorType[..., "num_samples"]): Weights for each sample Returns: - RendererOutputs: Outputs with rgb values. + TensorType[..., 3]: Outputs of rgb values. """ sh = sh.view(*sh.shape[:-1], 3, sh.shape[-1] // 3) @@ -125,8 +111,7 @@ class SHRenderer(nn.Module): rgb = RGBRenderer.combine_rgb(rgb, weights, background_color=self.background_color) - renderer_outputs = RendererOutputs(rgb=rgb) - return renderer_outputs + return rgb class AccumulationRenderer(nn.Module): @@ -136,20 +121,18 @@ class AccumulationRenderer(nn.Module): def forward( cls, weights: TensorType[..., "num_samples"], - ) -> RendererOutputs: + ) -> TensorType: """Composite samples along ray and calculate accumulation. Args: weights (TensorType[..., "num_samples"]): Weights for each sample Returns: - RendererOutputs: Outputs with accumulated values. + TensorType: Outputs of accumulated values. """ accumulation = torch.sum(weights, dim=-1)[..., None] - - renderer_outputs = RendererOutputs(accumulation=accumulation) - return renderer_outputs + return accumulation class DepthRenderer(nn.Module): @@ -165,7 +148,9 @@ class DepthRenderer(nn.Module): raise ValueError(f"{method} is an invalid depth calculation method") self.method = method - def forward(self, weights: TensorType[..., "num_samples"], ts: TensorType[..., "num_samples"]) -> RendererOutputs: + def forward( + self, weights: TensorType[..., "num_samples"], ts: TensorType[..., "num_samples"] + ) -> TensorType[..., 1]: """Composite samples along ray and calculate disparities. Args: @@ -173,7 +158,7 @@ class DepthRenderer(nn.Module): ts (TensorType[..., "num_samples"]): Sample locations along rays Returns: - RendererOutputs: Outputs with depth values. + TensorType[..., 1]: Outputs of depth values. """ if self.method == "expected": @@ -182,7 +167,7 @@ class DepthRenderer(nn.Module): depth = torch.clip(depth, ts[..., 0], ts[..., -1]) - return RendererOutputs(depth=depth[..., None]) + return depth[..., None] raise NotImplementedError(f"Method {self.method} not implemented") @@ -191,9 +176,7 @@ class UncertaintyRenderer(nn.Module): """Calculate uncertainty along the ray.""" @classmethod - def forward( - cls, betas: TensorType[..., "num_samples", 1], weights: TensorType[..., "num_samples"] - ) -> RendererOutputs: + def forward(cls, betas: TensorType[..., "num_samples", 1], weights: TensorType[..., "num_samples"]) -> TensorType: """_summary_ Args: @@ -201,11 +184,10 @@ class UncertaintyRenderer(nn.Module): weights (TensorType[..., "num_samples"]): _description_ Returns: - RendererOutputs: _description_ + TensorType: _description_ """ uncertainty = torch.sum(weights[..., None] * betas, dim=-2) - renderer_outputs = RendererOutputs(uncertainty=uncertainty) - return renderer_outputs + return uncertainty class SemanticRenderer(nn.Module): @@ -214,8 +196,7 @@ class SemanticRenderer(nn.Module): @classmethod def forward( cls, semantics: TensorType[..., "num_samples", "num_classes"], weights: TensorType[..., "num_samples"] - ) -> RendererOutputs: + ) -> TensorType[..., "num_classes"]: """_summary_""" sem = torch.sum(weights[..., None] * semantics, dim=-2) - renderer_outputs = RendererOutputs(semantics=sem) - return renderer_outputs + return sem diff --git a/mattport/nerf/trainer.py b/mattport/nerf/trainer.py index a612609c4..ec2486a92 100644 --- a/mattport/nerf/trainer.py +++ b/mattport/nerf/trainer.py @@ -246,7 +246,7 @@ class Trainer: """Run one iteration with a batch of inputs.""" # move batch to correct device ray_indices = batch["indices"].to(self.device) - _, loss_dict = self.graph(ray_indices, batch=batch) + _, loss_dict = self.graph(ray_indices, batch=batch, step=step) loss = loss_dict["aggregated_loss"] self.optimizers.zero_grad_all() loss.backward() diff --git a/mattport/structures/rays.py b/mattport/structures/rays.py index 8f5697a6d..d1df55b97 100644 --- a/mattport/structures/rays.py +++ b/mattport/structures/rays.py @@ -11,6 +11,15 @@ from torchtyping import TensorType from mattport.utils.misc import is_not_none +@dataclass +class PointSamples: + """Samples in space.""" + + positions: TensorType[..., 3] = None + directions: TensorType[..., 3] = None + camera_indices: TensorType[..., 1] = None + + @dataclass class RayBundle: """_summary_ @@ -137,6 +146,11 @@ class RaySamples: self.directions = ray_bundle.directions.unsqueeze(1).repeat(1, self.positions.shape[1], 1) self.deltas = self.get_deltas() + def to_point_samples(self): + """Convert to PointSamples instance and return.""" + # TODO: make this more interpretable + return PointSamples(positions=self.positions, directions=self.directions) + def get_camera_indices(self): """Returns camera indices.""" assert not isinstance(self.ray_bundle.camera_indices, type(None)), "Camera indices cannot be None here." diff --git a/mattport/utils/io.py b/mattport/utils/io.py index 445a0c3fe..1afd26e48 100644 --- a/mattport/utils/io.py +++ b/mattport/utils/io.py @@ -4,6 +4,8 @@ Input/output utils. import json import os +import logging + def load_from_json(filename: str): """_summary_ @@ -51,16 +53,23 @@ def get_git_root(path, dirs=(".git",), default=None): return default -def get_project_root(): +def get_project_root(path): """Return the project root directory from an environment variable. # TODO: handle this better and report user error """ project_root = os.getenv("PROJECT_ROOT") - assert project_root is not None, "Please set PROJECT_ROOT to the root directory of this repo." + if project_root is None: + logging.info( + ( + "Please set PROJECT_ROOT to the root directory of this repo. " + "Going to try calling get_git_root(path) instead." + ) + ) + project_root = get_git_root(path) return project_root -def get_absolute_path(path, proj_root_func=get_git_root): +def get_absolute_path(path, proj_root_func=get_project_root): """ Returns the full, absolute path. Relative paths are assumed to start at the repo directory. diff --git a/mattport/utils/stats_tracker.py b/mattport/utils/stats_tracker.py index 066a9baa9..64f0fc5f5 100644 --- a/mattport/utils/stats_tracker.py +++ b/mattport/utils/stats_tracker.py @@ -157,13 +157,13 @@ class StatsTracker: v = f"{v:0.4f}" curr_mssg += f"{v:<20} " # update the history buffer - if len(self.past_stats) >= self.max_history: - self.past_stats.pop(0) - cursorup(len(self.past_stats) + 1) - for mssg in self.past_stats: - print(mssg) + # if len(self.past_stats) >= self.max_history: + # self.past_stats.pop(0) + # cursorup(len(self.past_stats) + 1) + # for mssg in self.past_stats: + # print(mssg) print(curr_mssg) - self.past_stats.append(curr_mssg) + # self.past_stats.append(curr_mssg) @check_print_stats_step def print_stats(self, fraction_done: float): @@ -173,7 +173,7 @@ class StatsTracker: fraction_done (float): fraction of steps executed in training iterations """ # print a new header line if there is a new key added - self.handle_header() + # self.handle_header() # generate a new stats reporting message self.handle_stats(fraction_done) diff --git a/pyproject.toml b/pyproject.toml index 5cd8755fb..258f0db85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ line-length = 120 max-line-length = 120 generated-members = ["numpy.*", "torch.*", "cv2.*", "cv.*"] good-names-rgxs = "^[_a-zA-Z][_a-z0-9]?$" -ignore = ["mattport/cuda", "tests/cuda"] +ignore = ["mattport/cuda", "tests/cuda", "nerfw.py", "semantic_nerf.py"] disable = [ "too-many-arguments", diff --git a/setup.py b/setup.py index 78db55ab3..b7ceb0cd1 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ import os from setuptools import setup ROOT_DIR = os.path.dirname(os.path.abspath(__file__)) -USE_CUDA = False +USE_CUDA = True ext_modules = [] cmdclass = {} diff --git a/tests/nerf/dataloader/test_ray_generator.py b/tests/nerf/dataloader/test_ray_generator.py index cd311cc98..022a169b3 100644 --- a/tests/nerf/dataloader/test_ray_generator.py +++ b/tests/nerf/dataloader/test_ray_generator.py @@ -10,7 +10,7 @@ from tqdm import tqdm from mattport.nerf.dataset.collate import CollateIterDataset, collate_batch_size_one from mattport.nerf.dataset.image_dataset import ImageDataset, collate_batch from mattport.nerf.dataset.utils import get_dataset_inputs -from mattport.nerf.field_modules.ray_generator import RayGenerator +from mattport.nerf.ray_generator import RayGenerator from mattport.utils.io import get_absolute_path from mattport.viewer.plotly import visualize_dataset diff --git a/tests/nerf/field_modules/test_field_outputs.py b/tests/nerf/field_modules/test_field_outputs.py index 3175ea673..fcc07fec8 100644 --- a/tests/nerf/field_modules/test_field_outputs.py +++ b/tests/nerf/field_modules/test_field_outputs.py @@ -7,7 +7,6 @@ from torch import nn from mattport.nerf.field_modules.field_heads import ( DensityFieldHead, FieldHead, - FieldHeadNames, RGBFieldHead, SHFieldHead, ) @@ -17,9 +16,8 @@ def test_field_output(): """Test render output""" in_dim = 6 out_dim = 4 - field_head_name = FieldHeadNames.DENSITY activation = nn.ReLU() - render_head = FieldHead(in_dim=in_dim, out_dim=out_dim, field_head_name=field_head_name, activation=activation) + render_head = FieldHead(in_dim=in_dim, out_dim=out_dim, activation=activation) assert render_head.get_out_dim() == out_dim x = torch.ones((9, in_dim)) diff --git a/tests/nerf/test_occupancy_grid.py b/tests/nerf/test_occupancy_grid.py new file mode 100644 index 000000000..b7c33f61a --- /dev/null +++ b/tests/nerf/test_occupancy_grid.py @@ -0,0 +1,22 @@ +""" +Code to test the occupancy grid. +""" + +import torch +from mattport.nerf.occupancy_grid import OccupancyGrid + + +def test_occupancy_grid(): + """For testing the occupancy grid.""" + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + num_cascades = 1 + resolution = 128 + aabb = torch.tensor([[-1, -1, -1], [1, 1, 1]], dtype=torch.float32).to(device) + grid = OccupancyGrid(num_cascades, resolution, aabb) + grid.to(device) + density_fn = lambda x: x[..., 0:1] # xyz (..., 3) => density (...,) + grid.update_occupancy_grid(density_fn=density_fn) + + +if __name__ == "__main__": + test_occupancy_grid() diff --git a/tests/nerf/test_renderers.py b/tests/nerf/test_renderers.py index 4cbf6b147..4d1e89ffc 100644 --- a/tests/nerf/test_renderers.py +++ b/tests/nerf/test_renderers.py @@ -11,17 +11,17 @@ def test_rgb_renderer(): """Test RGB volumetric rendering""" num_samples = 10 - rgb = torch.ones((3, num_samples, 3)) + rgb_samples = torch.ones((3, num_samples, 3)) weights = torch.ones((3, num_samples)) weights /= torch.sum(weights, axis=-1, keepdim=True) rgb_renderer = renderers.RGBRenderer() - out = rgb_renderer(rgb=rgb, weights=weights) - assert torch.max(out.rgb) > 0.9 + rgb = rgb_renderer(rgb=rgb_samples, weights=weights) + assert torch.max(rgb) > 0.9 - out = rgb_renderer(rgb=rgb * 0, weights=weights) - assert torch.max(out.rgb) == pytest.approx(0) + rgb = rgb_renderer(rgb=rgb_samples * 0, weights=weights) + assert torch.max(rgb) == pytest.approx(0) def test_sh_renderer(): @@ -38,8 +38,8 @@ def test_sh_renderer(): sh_renderer = renderers.SHRenderer() - out = sh_renderer(sh=sh, directions=directions, weights=weights) - assert torch.max(out.rgb) > 0.9 + rgb = sh_renderer(sh=sh, directions=directions, weights=weights) + assert torch.max(rgb) > 0.9 def test_acc_renderer(): @@ -51,8 +51,8 @@ def test_acc_renderer(): acc_renderer = renderers.AccumulationRenderer() - out = acc_renderer(weights=weights) - assert torch.max(out.accumulation) > 0.9 + accumulation = acc_renderer(weights=weights) + assert torch.max(accumulation) > 0.9 def test_depth_renderer(): @@ -67,8 +67,8 @@ def test_depth_renderer(): depth_renderer = renderers.DepthRenderer() - out = depth_renderer(weights=weights, ts=ts) - assert torch.min(out.depth) > 0 + depth = depth_renderer(weights=weights, ts=ts) + assert torch.min(depth) > 0 if __name__ == "__main__": diff --git a/tests/nerf/test_sampler.py b/tests/nerf/test_sampler.py index f69e43b8d..92a6848bb 100644 --- a/tests/nerf/test_sampler.py +++ b/tests/nerf/test_sampler.py @@ -5,7 +5,7 @@ import torch from mattport.nerf.sampler import PDFSampler, UniformSampler from mattport.structures.rays import RayBundle -from mattport.nerf.field_modules.scene_bounds_collider import NearFarCollider +from mattport.nerf.colliders import NearFarCollider def test_uniform_sampler():