Adios mattport

This commit is contained in:
Matt Tancik committed 2022-05-30 09:24:12 -07:00
1 parent 9d2b34ab60
commit b26408eb7a
86 files changed
+176 -176

No files matched your search

View File
Whitespace-only changes.
+269
View File
@@ -0,0 +1,269 @@
"""
Camera Models
"""
from abc import abstractmethod
from dataclasses import dataclass
from typing import Optional, Type
import torch
from torch.nn.functional import normalize
from torchtyping import TensorType
from radiance.structures.rays import RayBundle
@dataclass
class Rays:
"""Camera rays. Can be arbitrary dimension"""
origin: TensorType[..., 3]
direction: TensorType[..., 3]
class Camera:
"""Base Camera. Intended to be subclassed"""
def __init__(self, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]) -> None:
self.camera_to_world = camera_to_world
@abstractmethod
def get_num_intrinsics_params(self) -> int:
"""
Returns:
int: number of optimizable intrinsic parameters
"""
return
@abstractmethod
def get_intrinsics(self) -> torch.Tensor:
"""
Returns:
torch.Tensor: Intrinsics matrix
"""
return
def get_camera_to_world(self) -> TensorType[3, 4]:
"""
Returns:
int: Camera to world transformation
"""
return self.camera_to_world
@abstractmethod
def get_image_height(self) -> int:
"""
Returns:
int: Image height
"""
return
@abstractmethod
def get_image_width(self) -> int:
"""
Returns:
int: Image width
"""
return
def get_image_coords(self, pixel_offset: float = 0.5) -> TensorType["image_height", "image_width", 2]:
"""
Args:
pixel_offset (float): Offset for each pixel. Defaults to center of pixel (0.5)
Returns:
TensorType["image_height", "image_width", 2]: Grid of image coordinates.
"""
image_height = self.get_image_height()
image_width = self.get_image_width()
image_coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij")
image_coords = torch.stack(image_coords, dim=-1) + pixel_offset # stored as (y, x) coordinates
return image_coords
@classmethod
@abstractmethod
def generate_rays(
cls,
intrinsics: TensorType["num_rays", "num_intrinsics_params"],
camera_to_world: TensorType["num_rays", 3, 4],
coords: TensorType["image_height", "image_width", 2],
) -> RayBundle:
"""
Args:
intrinsics (TensorType["num_rays", "num_intrinsics_params"]): Camera intrinsics
camera_to_world (TensorType["num_rays", 3, 4]): Camera to world transformation matrix
coords (TensorType["image_height", "image_width", 2]): Image grid coordinates
Returns:
RayBundle: A bundle of rays for each grid coordinate.
"""
return
def generate_camera_rays(self) -> RayBundle:
"""Generate rays for the camera.
Returns:
Rays: Camera rays of shape [height, width]
"""
height = self.get_image_height()
width = self.get_image_width()
intrinsics = self.get_intrinsics().unsqueeze(0).repeat(height, width, 1) # (num_rays, num_intrinsics_params)
camera_to_world = self.camera_to_world.unsqueeze(0).repeat(height, width, 1, 1) # (num_rays, 3, 4)
coords = self.get_image_coords()
ray_bundle = self.generate_rays(intrinsics, camera_to_world, coords)
return ray_bundle
class PinholeCamera(Camera):
"""Pinhole camera model."""
def __init__(
self, cx: float, cy: float, fx: float, fy: float, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]
):
super().__init__(camera_to_world)
self.cx = cx
self.cy = cy
self.fx = fx
self.fy = fy
def get_num_intrinsics_params(self) -> int:
return 4
def get_image_width(self) -> int:
return int(self.cx * 2.0)
def get_image_height(self) -> int:
return int(self.cy * 2.0)
def get_intrinsics_matrix(self) -> TensorType[3, 3]:
"""
Returns:
TensorType[3, 3]: Pinhole camera intrinsics matrix
"""
K = torch.tensor(
[[self.fx, 0.0, self.cx], [0.0, self.fy, self.cy], [0, 0, 1.0]],
dtype=torch.float32,
)
return K
def get_intrinsics(self) -> torch.Tensor:
return torch.tensor([self.cx, self.cy, self.fx, self.fy])
@classmethod
def fx_index(cls):
"""TODO(ethan): redo this in a better way.
Ideally we can dynamically grab the focal length parameters depending on
Simple vs. not Simple Pinhole Model.
Returns:
_type_: _description_
"""
return 2
@classmethod
def fy_index(cls):
"""_summary_
Returns:
_type_: _description_
"""
return 3
@classmethod
def generate_rays(
cls,
intrinsics: TensorType[..., "num_intrinsics_params"],
camera_to_world: TensorType[..., 3, 4],
coords: TensorType[..., 2],
) -> RayBundle:
cx = intrinsics[..., 0:1]
cy = intrinsics[..., 1:2]
fx = intrinsics[..., cls.fx_index() : cls.fx_index() + 1]
fy = intrinsics[..., cls.fy_index() : cls.fy_index() + 1]
y = coords[..., 0:1] # (..., 1)
x = coords[..., 1:2] # (..., 1)
original_directions = torch.cat([(x - cx) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1) # (..., 3)
rotation = camera_to_world[..., :3, :3] # (..., 3, 3)
directions = torch.sum(
original_directions[..., None, :] * rotation, dim=-1
) # (..., 1, 3) * (..., 3, 3) -> (..., 3)
directions = normalize(directions, dim=-1)
origins = camera_to_world[..., :3, 3] # (..., 3)
return RayBundle(origins=origins, directions=directions)
class SimplePinholeCamera(PinholeCamera):
"""Simple Pinhole Camera model."""
def __init__(self, cx: float, cy: float, f: float, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]):
super().__init__(cx, cy, f, f, camera_to_world)
@classmethod
def fx_index(cls):
return 2
@classmethod
def fy_index(cls):
return 2
class EquirectangularCamera(Camera):
"""Equirectangular (360 degree) camera model."""
def __init__(self, height: int, width: int, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]):
super().__init__(camera_to_world)
self.height = height
self.width = width
def get_num_intrinsics_params(self):
return 2
def get_image_width(self):
return self.width
def get_image_height(self):
return self.height
def get_intrinsics(self) -> torch.Tensor:
return torch.tensor([self.height, self.width])
@classmethod
def generate_rays(
cls,
intrinsics: TensorType["num_rays", 2],
camera_to_world: TensorType["num_rays", 3, 4],
coords: TensorType["num_rays", 2],
) -> RayBundle:
phi = coords[:, 0:1] # (num_rays, 1)
theta = coords[:, 1:2] # (num_rays, 1)
height = intrinsics[:, 0:1]
width = intrinsics[:, 1:2]
phi = phi / height * torch.pi
theta = -theta / width * 2 * torch.pi
directions = torch.stack(
[torch.cos(theta) * torch.sin(phi), torch.sin(theta) * torch.sin(phi), torch.cos(phi)], dim=-1
)
origins = camera_to_world[:, :3, 3] # (num_rays, 3)
return RayBundle(origins=origins, directions=directions)
def get_camera_model(num_intrinsics_params: int) -> Type[Camera]:
"""Returns the camera model given the specified number of intrinsics parameters.
Args:
num_intrinsics_params (int): Number of intrinsic parametes.
Returns:
Camera: Camera model
"""
if num_intrinsics_params == 2:
return EquirectangularCamera
if num_intrinsics_params == 3:
return SimplePinholeCamera
if num_intrinsics_params == 4:
return PinholeCamera
raise NotImplementedError
+40
View File
@@ -0,0 +1,40 @@
"""Common Colors"""
from typing import Union
from omegaconf import ListConfig
import torch
from torchtyping import TensorType
WHITE = torch.tensor([1.0, 1.0, 1.0])
BLACK = torch.tensor([0.0, 0.0, 0.0])
RED = torch.tensor([1.0, 0.0, 0.0])
GREEN = torch.tensor([0.0, 1.0, 0.0])
BLUE = torch.tensor([0.0, 0.0, 1.0])
COLORS_DICT = {
"white": WHITE,
"black": BLACK,
"red": RED,
"green": GREEN,
"blue": BLUE,
}
def get_color(color: Union[str, list, ListConfig]) -> TensorType[3]:
"""
Args:
color (Union[str, list, ListConfig]): Color as a string or a rgb list
Returns:
TensorType[3]: Parsed color
"""
if isinstance(color, str):
color = color.lower()
if color not in COLORS_DICT:
raise ValueError(f"{color} is not a valid preset color")
return COLORS_DICT[color]
if isinstance(color, (list, ListConfig)):
if len(color) != 3:
raise ValueError(f"Color should be 3 values (RGB) instead got {color}")
return torch.tensor(color)
raise ValueError(f"Color should be an RGB list or string, instead got {type(color)}")
+244
View File
@@ -0,0 +1,244 @@
"""
Some ray datastructures.
"""
import random
from dataclasses import dataclass
from typing import Optional
import torch
from torchtyping import TensorType
from radiance.utils.misc import is_not_none
@dataclass
class PointSamples:
"""Samples in space.
Args:
positions (TensorType[..., 3]): XYZ locations
directions (TensorType[..., 3]): Unit direction vector
camera_indices (TensorType[..., 1]): Camera index
valid_mask (TensorType[...]): Rays that are valid
"""
positions: TensorType[..., 3]
directions: TensorType[..., 3] = None
camera_indices: TensorType[..., 1] = None
valid_mask: TensorType[...] = None
def apply_masks(self) -> "PointSamples":
"""Use valid_mask to mask samples.
Returns:
PointSamples: New set of masked samples.
"""
if is_not_none(self.valid_mask):
positions = self.positions[self.valid_mask]
directions = self.directions[self.valid_mask] if is_not_none(self.directions) else self.directions
camera_indices = (
self.camera_indices[self.valid_mask] if is_not_none(self.camera_indices) else self.camera_indices
)
return PointSamples(
positions=positions,
directions=directions,
camera_indices=camera_indices,
)
return PointSamples(
positions=self.positions,
directions=self.directions,
camera_indices=self.camera_indices,
)
@dataclass
class RaySamples:
"""Samples along a ray
Args:
positions (TensorType[..., 3]): XYZ locations
directions (TensorType[..., 3]): Unit direction vector
camera_indices (TensorType[..., 1]): Camera index
valid_mask (TensorType[...]): Rays that are valid
ts (TensorType[..., 1]): "time steps", distances along ray
deltas )TensorType[..., 1]): "width" of each sample
"""
positions: TensorType[..., 3]
directions: TensorType[..., 3] = None
camera_indices: TensorType[..., 1] = None
valid_mask: TensorType[...] = None
ts: TensorType[..., 1] = None
deltas: TensorType[..., 1] = None
def to_point_samples(self) -> PointSamples:
"""Convert to PointSamples instance and return."""
# TODO: make this more interpretable
return PointSamples(positions=self.positions, directions=self.directions, valid_mask=self.valid_mask)
def get_weights(self, densities: TensorType[..., "num_samples", 1]) -> TensorType[..., "num_samples"]:
"""Return weights based on predicted densities
Args:
densities (TensorType[..., "num_samples", 1]): Predicted densities for samples along ray
Returns:
TensorType[..., "num_samples"]: Weights for each sample
"""
delta_density = self.deltas * densities[..., 0]
alphas = 1 - torch.exp(-delta_density)
# mip-nerf version of transmittance calculation:
transmittance = torch.cumsum(delta_density[..., :-1], dim=-1)
transmittance = torch.cat(
[torch.zeros((*transmittance.shape[:1], 1)).to(densities.device), transmittance], axis=-1
)
transmittance = torch.exp(-transmittance) # [..., "num_samples"]
# most nerf codebases do the following:
# transmittance = torch.cat(
# [torch.ones((*alphas.shape[:1], 1)).to(densities.device), 1.0 - alphas + 1e-10], dim=-1
# )
# transmittance = torch.cumprod(transmittance, dim=-1)[..., :-1] # [..., "num_samples"]
weights = alphas * transmittance # [..., "num_samples"]
return weights
def set_valid_mask(self, valid_mask: TensorType[..., "num_samples"]) -> None:
"""Sets valid mask"""
self.valid_mask = valid_mask
@dataclass
class RayBundle:
"""A bundle of ray parameters.
Args:
origins (TensorType[..., 3]): Ray origins (XYZ)
directions (TensorType[..., 3]): Unit ray direction vector
camera_indices (TensorType[..., 1]): Camera indices
nears (TensorType[..., 1]): Distance along ray to start sampling
fars (TensorType[..., 1]): Rays Distance along ray to stop sampling
valid_mask (TensorType[...]): Rays that are valid
"""
origins: TensorType["num_rays", 3]
directions: TensorType["num_rays", 3]
camera_indices: Optional[TensorType["num_rays", 1]] = None
nears: Optional[TensorType["num_rays"]] = None
fars: Optional[TensorType["num_rays"]] = None
valid_mask: Optional[TensorType["num_rays"]] = None
def move_to_device(self, device: torch.device) -> None:
"""Move bundle data to a device.
Args:
device (torch.device): Device to move tensors to.
"""
self.origins = self.origins.to(device)
self.directions = self.directions.to(device)
if not isinstance(self.camera_indices, type(None)):
self.camera_indices = self.camera_indices.to(device)
def set_camera_indices(self, camera_index: int) -> None:
"""Sets all of the the camera indices to a specific camera index.
Args:
camera_index (int): Camera index.
"""
self.camera_indices = torch.ones_like(self.origins[..., 0]).long() * camera_index
def __len__(self):
num_rays = torch.numel(self.origins) // self.origins.shape[-1]
return num_rays
def sample(self, num_rays: int) -> "RayBundle":
"""Returns a RayBundle as a subset of rays.
Args:
num_rays (int): Number of rays in output RayBundle
Returns:
RayBundle: RayBundle with subset of rays.
"""
assert num_rays <= len(self)
indices = random.sample(range(len(self)), k=num_rays)
return RayBundle(
origins=self.origins[indices],
directions=self.directions[indices],
camera_indices=self.camera_indices[indices],
)
def get_masked_ray_bundle(self, valid_mask: TensorType) -> "RayBundle":
"""Return a masked instance of the ray bundle.
Args:
valid_mask (TensorType): Mask of rays to keep.
Returns:
RayBundle: RayBundle of masked rays.
"""
return RayBundle(
origins=self.origins[valid_mask],
directions=self.directions[valid_mask],
camera_indices=self.camera_indices[valid_mask] if is_not_none(self.camera_indices) else None,
nears=self.nears[valid_mask] if is_not_none(self.nears) else None,
fars=self.fars[valid_mask] if is_not_none(self.fars) else None,
valid_mask=self.valid_mask[valid_mask] if is_not_none(self.valid_mask) else None,
)
def get_row_major_sliced_ray_bundle(self, start_idx: int, end_idx: int) -> "RayBundle":
"""Flattens RayBundle and extracts chunk given start and end indicies.
Args:
start_idx (int): Start index of RayBundle chunk.
end_idx (int): End index of RayBundle chunk.
Returns:
RayBundle: Flattened RayBundle with end_idx-start_idx rays.
"""
camera_indices = (
self.camera_indices.view(-1)[start_idx:end_idx] if not isinstance(self.camera_indices, type(None)) else None
)
return RayBundle(
origins=self.origins.view(-1, 3)[start_idx:end_idx],
directions=self.directions.view(-1, 3)[start_idx:end_idx],
camera_indices=camera_indices,
)
def get_ray_samples(self, ts: TensorType[..., "num_samples"]) -> RaySamples:
"""Produces samples for each ray by projection points along the ray direction.
Args:
ts (TensorType[..., "num_samples"]): Distance from origin of sample points.
Returns:
RaySamples: Samples projected along ray.
"""
positions = self.origins[:, None] + ts[:, :, None] * self.directions[:, None]
directions = self.directions.unsqueeze(1).repeat(1, positions.shape[1], 1)
valid_mask = torch.ones_like(ts, dtype=torch.bool)
dists = ts[..., 1:] - ts[..., :-1]
dists = torch.cat([dists, dists[..., -1:]], -1) # [N_rays, N_samples]
deltas = dists * torch.norm(self.directions[..., None, :], dim=-1)
if self.camera_indices is not None:
camera_indices = self.camera_indices.unsqueeze(1).repeat(1, positions.shape[1])
else:
camera_indices = None
ray_samples = RaySamples(
positions=positions,
directions=directions,
camera_indices=camera_indices,
valid_mask=valid_mask,
ts=ts,
deltas=deltas,
)
return ray_samples