nerfstudio/pyrad/fields/occupancy_fields/occupancy_grid.py

# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Code to implement the density grid.
"""

import torch
import torch.nn.functional as F
from torch import nn
from torchtyping import TensorType

from pyrad.utils.misc import is_not_none


class OccupancyGrid(nn.Module):
    """Module to keep track of the density and occupancy."""

    def __init__(
        self, num_cascades: int = 1, resolution: int = 128, aabb=None, update_every_num_iters: int = 16
    ) -> None:
        super().__init__()
        assert is_not_none(aabb), "The axis-aligned bounding box aabb is not defined!"
        self.num_cascades = num_cascades  # the number of levels (i.e, cascades)
        self.resolution = resolution
        self.register_buffer("aabb", aabb)  # axis-aligned bounding box
        occupancy_grid = torch.ones(
            [self.num_cascades] + [self.resolution] * 3
        )  # (num_cascades, x_resolution, y_resolution, z_resolution)
        # not a module parameter, but it still should be saved and part of `state_dict`
        self.register_buffer("occupancy_grid", occupancy_grid)
        self.mean_density = 0.0

        self.update_every_num_iters = update_every_num_iters

    def reset(self):
        """Zero out the occupancy grid."""
        self.occupancy_grid.zero_()

    @torch.no_grad()
    def mark_untrained_grid(self):
        """Marks the locations in the density grid that are never seen by training views."""
        # TODO: only query points that are covered by the training cameras viewing frustrum
        # We'll need to pass in the camera poses and intrinsics to this function
        raise NotImplementedError()

    @torch.no_grad()
    def update_occupancy_grid(self, density_fn, decay=0.95, split_size=64):
        """Update the occupancy grid with density values.

        Args:
            density_fn: Function from takes points in the shape (..., 3) and
            returns a tensor of densities of shape (..., 1).
        """
        # TODO(ethan): remove this pylint disable
        #  pylint: disable=invalid-name
        device = self.occupancy_grid.device

        assert self.num_cascades == 1, "Still need to implement code for multiple cascades."
        temp_occupancy_grid = torch.zeros_like(self.occupancy_grid)

        X_length, Y_length, Z_length = self.aabb[1] - self.aabb[0]
        X_half_voxel_length = 0.5 * X_length / self.resolution
        Y_half_voxel_length = 0.5 * Y_length / self.resolution
        Z_half_voxel_length = 0.5 * Z_length / self.resolution
        half_voxel_lengths = torch.tensor(
            [X_half_voxel_length, Y_half_voxel_length, Z_half_voxel_length], device=device
        )
        X = torch.linspace(
            self.aabb[0][0] + X_half_voxel_length, self.aabb[1][0] - X_half_voxel_length, self.resolution, device=device
        ).split(split_size)
        Y = torch.linspace(
            self.aabb[0][1] + Y_half_voxel_length, self.aabb[1][1] - Y_half_voxel_length, self.resolution, device=device
        ).split(split_size)
        Z = torch.linspace(
            self.aabb[0][2] + Z_half_voxel_length, self.aabb[1][2] - Z_half_voxel_length, self.resolution, device=device
        ).split(split_size)
        for x_index, xs in enumerate(X):
            for y_index, ys in enumerate(Y):
                for z_index, zs in enumerate(Z):
                    x_len, y_len, z_len = len(xs), len(ys), len(zs)
                    grid_x, grid_y, grid_z = torch.meshgrid(xs, ys, zs, indexing="ij")
                    xyzs = torch.stack([grid_x, grid_y, grid_z], dim=-1)  # (resolution, resolution,, resolution, 3)
                    # add noise to the points
                    xyzs_rand = (
                        torch.rand_like(xyzs) * half_voxel_lengths
                    )  # range [-half_voxel_length, half_voxel_length] for each dimension
                    xyzs += xyzs_rand
                    densities = density_fn(xyzs)
                    assert densities.shape[-1] == 1
                    temp_occupancy_grid[
                        0,
                        x_index * split_size : x_index * split_size + x_len,
                        y_index * split_size : y_index * split_size + y_len,
                        z_index * split_size : z_index * split_size + z_len,
                    ] = densities[..., 0]

        # exponential moving average update
        valid_mask = self.occupancy_grid >= 0.0
        self.occupancy_grid[valid_mask] = torch.maximum(
            self.occupancy_grid[valid_mask] * decay, temp_occupancy_grid[valid_mask]
        )
        self.mean_density = torch.mean(self.occupancy_grid[valid_mask]).item()

    @torch.no_grad()
    def get_densities(self, xyzs: TensorType[..., 3]) -> TensorType[..., 1]:
        """Trilinear interpolation to get the density values.

        Args:
            xyzs (TensorType[..., 3]): 3D querry coordinate

        Returns:
            TensorType[..., 1]: Density values
        """
        occupancy_grid = self.occupancy_grid[None, ...]  # shape (1, num_cascades, X_res, Y_res, Z_res)
        xyzs_shape = xyzs.shape
        xyzs_reshaped = xyzs.view(1, -1, 1, 1, 3)
        voxel_lengths = self.aabb[1] - self.aabb[0]
        xyzs_reshaped_normalized = ((xyzs_reshaped - self.aabb[0]) / voxel_lengths) * 2.0 - 1.0
        densities = F.grid_sample(
            occupancy_grid.permute(
                0, 1, 4, 3, 2
            ),  # (xyz to zyx) for grid sample useage because the xyzs have xyz ordering
            xyzs_reshaped_normalized,
            align_corners=True,
            padding_mode="zeros",
        )
        densities = densities.view(*xyzs_shape[:-1], 1)
        return densities

    def forward(self, x):
        """Not implemented."""
        raise NotImplementedError()