mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 07:05:44 +08:00
Adios mattport
This commit is contained in:
1 parent
9d2b34ab60
commit
b26408eb7a
86 files changed
+176
-176
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,269 @@
|
||||
"""
|
||||
Camera Models
|
||||
"""
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Type
|
||||
|
||||
import torch
|
||||
from torch.nn.functional import normalize
|
||||
from torchtyping import TensorType
|
||||
|
||||
from radiance.structures.rays import RayBundle
|
||||
|
||||
|
||||
@dataclass
|
||||
class Rays:
|
||||
"""Camera rays. Can be arbitrary dimension"""
|
||||
|
||||
origin: TensorType[..., 3]
|
||||
direction: TensorType[..., 3]
|
||||
|
||||
|
||||
class Camera:
|
||||
"""Base Camera. Intended to be subclassed"""
|
||||
|
||||
def __init__(self, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]) -> None:
|
||||
self.camera_to_world = camera_to_world
|
||||
|
||||
@abstractmethod
|
||||
def get_num_intrinsics_params(self) -> int:
|
||||
"""
|
||||
Returns:
|
||||
int: number of optimizable intrinsic parameters
|
||||
"""
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def get_intrinsics(self) -> torch.Tensor:
|
||||
"""
|
||||
Returns:
|
||||
torch.Tensor: Intrinsics matrix
|
||||
"""
|
||||
return
|
||||
|
||||
def get_camera_to_world(self) -> TensorType[3, 4]:
|
||||
"""
|
||||
Returns:
|
||||
int: Camera to world transformation
|
||||
"""
|
||||
return self.camera_to_world
|
||||
|
||||
@abstractmethod
|
||||
def get_image_height(self) -> int:
|
||||
"""
|
||||
Returns:
|
||||
int: Image height
|
||||
"""
|
||||
return
|
||||
|
||||
@abstractmethod
|
||||
def get_image_width(self) -> int:
|
||||
"""
|
||||
Returns:
|
||||
int: Image width
|
||||
"""
|
||||
return
|
||||
|
||||
def get_image_coords(self, pixel_offset: float = 0.5) -> TensorType["image_height", "image_width", 2]:
|
||||
"""
|
||||
Args:
|
||||
pixel_offset (float): Offset for each pixel. Defaults to center of pixel (0.5)
|
||||
|
||||
Returns:
|
||||
TensorType["image_height", "image_width", 2]: Grid of image coordinates.
|
||||
"""
|
||||
image_height = self.get_image_height()
|
||||
image_width = self.get_image_width()
|
||||
image_coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij")
|
||||
image_coords = torch.stack(image_coords, dim=-1) + pixel_offset # stored as (y, x) coordinates
|
||||
return image_coords
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def generate_rays(
|
||||
cls,
|
||||
intrinsics: TensorType["num_rays", "num_intrinsics_params"],
|
||||
camera_to_world: TensorType["num_rays", 3, 4],
|
||||
coords: TensorType["image_height", "image_width", 2],
|
||||
) -> RayBundle:
|
||||
"""
|
||||
Args:
|
||||
intrinsics (TensorType["num_rays", "num_intrinsics_params"]): Camera intrinsics
|
||||
camera_to_world (TensorType["num_rays", 3, 4]): Camera to world transformation matrix
|
||||
coords (TensorType["image_height", "image_width", 2]): Image grid coordinates
|
||||
|
||||
Returns:
|
||||
RayBundle: A bundle of rays for each grid coordinate.
|
||||
"""
|
||||
return
|
||||
|
||||
def generate_camera_rays(self) -> RayBundle:
|
||||
"""Generate rays for the camera.
|
||||
|
||||
Returns:
|
||||
Rays: Camera rays of shape [height, width]
|
||||
"""
|
||||
height = self.get_image_height()
|
||||
width = self.get_image_width()
|
||||
intrinsics = self.get_intrinsics().unsqueeze(0).repeat(height, width, 1) # (num_rays, num_intrinsics_params)
|
||||
camera_to_world = self.camera_to_world.unsqueeze(0).repeat(height, width, 1, 1) # (num_rays, 3, 4)
|
||||
coords = self.get_image_coords()
|
||||
ray_bundle = self.generate_rays(intrinsics, camera_to_world, coords)
|
||||
return ray_bundle
|
||||
|
||||
|
||||
class PinholeCamera(Camera):
|
||||
"""Pinhole camera model."""
|
||||
|
||||
def __init__(
|
||||
self, cx: float, cy: float, fx: float, fy: float, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]
|
||||
):
|
||||
super().__init__(camera_to_world)
|
||||
self.cx = cx
|
||||
self.cy = cy
|
||||
self.fx = fx
|
||||
self.fy = fy
|
||||
|
||||
def get_num_intrinsics_params(self) -> int:
|
||||
return 4
|
||||
|
||||
def get_image_width(self) -> int:
|
||||
return int(self.cx * 2.0)
|
||||
|
||||
def get_image_height(self) -> int:
|
||||
return int(self.cy * 2.0)
|
||||
|
||||
def get_intrinsics_matrix(self) -> TensorType[3, 3]:
|
||||
"""
|
||||
Returns:
|
||||
TensorType[3, 3]: Pinhole camera intrinsics matrix
|
||||
"""
|
||||
K = torch.tensor(
|
||||
[[self.fx, 0.0, self.cx], [0.0, self.fy, self.cy], [0, 0, 1.0]],
|
||||
dtype=torch.float32,
|
||||
)
|
||||
return K
|
||||
|
||||
def get_intrinsics(self) -> torch.Tensor:
|
||||
return torch.tensor([self.cx, self.cy, self.fx, self.fy])
|
||||
|
||||
@classmethod
|
||||
def fx_index(cls):
|
||||
"""TODO(ethan): redo this in a better way.
|
||||
Ideally we can dynamically grab the focal length parameters depending on
|
||||
Simple vs. not Simple Pinhole Model.
|
||||
|
||||
Returns:
|
||||
_type_: _description_
|
||||
"""
|
||||
return 2
|
||||
|
||||
@classmethod
|
||||
def fy_index(cls):
|
||||
"""_summary_
|
||||
|
||||
Returns:
|
||||
_type_: _description_
|
||||
"""
|
||||
return 3
|
||||
|
||||
@classmethod
|
||||
def generate_rays(
|
||||
cls,
|
||||
intrinsics: TensorType[..., "num_intrinsics_params"],
|
||||
camera_to_world: TensorType[..., 3, 4],
|
||||
coords: TensorType[..., 2],
|
||||
) -> RayBundle:
|
||||
|
||||
cx = intrinsics[..., 0:1]
|
||||
cy = intrinsics[..., 1:2]
|
||||
fx = intrinsics[..., cls.fx_index() : cls.fx_index() + 1]
|
||||
fy = intrinsics[..., cls.fy_index() : cls.fy_index() + 1]
|
||||
y = coords[..., 0:1] # (..., 1)
|
||||
x = coords[..., 1:2] # (..., 1)
|
||||
original_directions = torch.cat([(x - cx) / fx, -(y - cy) / fy, -torch.ones_like(x)], -1) # (..., 3)
|
||||
rotation = camera_to_world[..., :3, :3] # (..., 3, 3)
|
||||
directions = torch.sum(
|
||||
original_directions[..., None, :] * rotation, dim=-1
|
||||
) # (..., 1, 3) * (..., 3, 3) -> (..., 3)
|
||||
directions = normalize(directions, dim=-1)
|
||||
origins = camera_to_world[..., :3, 3] # (..., 3)
|
||||
return RayBundle(origins=origins, directions=directions)
|
||||
|
||||
|
||||
class SimplePinholeCamera(PinholeCamera):
|
||||
"""Simple Pinhole Camera model."""
|
||||
|
||||
def __init__(self, cx: float, cy: float, f: float, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]):
|
||||
super().__init__(cx, cy, f, f, camera_to_world)
|
||||
|
||||
@classmethod
|
||||
def fx_index(cls):
|
||||
return 2
|
||||
|
||||
@classmethod
|
||||
def fy_index(cls):
|
||||
return 2
|
||||
|
||||
|
||||
class EquirectangularCamera(Camera):
|
||||
"""Equirectangular (360 degree) camera model."""
|
||||
|
||||
def __init__(self, height: int, width: int, camera_to_world: Optional[TensorType[3, 4]] = torch.eye(4)[:3]):
|
||||
super().__init__(camera_to_world)
|
||||
self.height = height
|
||||
self.width = width
|
||||
|
||||
def get_num_intrinsics_params(self):
|
||||
return 2
|
||||
|
||||
def get_image_width(self):
|
||||
return self.width
|
||||
|
||||
def get_image_height(self):
|
||||
return self.height
|
||||
|
||||
def get_intrinsics(self) -> torch.Tensor:
|
||||
return torch.tensor([self.height, self.width])
|
||||
|
||||
@classmethod
|
||||
def generate_rays(
|
||||
cls,
|
||||
intrinsics: TensorType["num_rays", 2],
|
||||
camera_to_world: TensorType["num_rays", 3, 4],
|
||||
coords: TensorType["num_rays", 2],
|
||||
) -> RayBundle:
|
||||
|
||||
phi = coords[:, 0:1] # (num_rays, 1)
|
||||
theta = coords[:, 1:2] # (num_rays, 1)
|
||||
|
||||
height = intrinsics[:, 0:1]
|
||||
width = intrinsics[:, 1:2]
|
||||
|
||||
phi = phi / height * torch.pi
|
||||
theta = -theta / width * 2 * torch.pi
|
||||
|
||||
directions = torch.stack(
|
||||
[torch.cos(theta) * torch.sin(phi), torch.sin(theta) * torch.sin(phi), torch.cos(phi)], dim=-1
|
||||
)
|
||||
origins = camera_to_world[:, :3, 3] # (num_rays, 3)
|
||||
return RayBundle(origins=origins, directions=directions)
|
||||
|
||||
|
||||
def get_camera_model(num_intrinsics_params: int) -> Type[Camera]:
|
||||
"""Returns the camera model given the specified number of intrinsics parameters.
|
||||
|
||||
Args:
|
||||
num_intrinsics_params (int): Number of intrinsic parametes.
|
||||
|
||||
Returns:
|
||||
Camera: Camera model
|
||||
"""
|
||||
if num_intrinsics_params == 2:
|
||||
return EquirectangularCamera
|
||||
if num_intrinsics_params == 3:
|
||||
return SimplePinholeCamera
|
||||
if num_intrinsics_params == 4:
|
||||
return PinholeCamera
|
||||
raise NotImplementedError
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Common Colors"""
|
||||
from typing import Union
|
||||
from omegaconf import ListConfig
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
|
||||
WHITE = torch.tensor([1.0, 1.0, 1.0])
|
||||
BLACK = torch.tensor([0.0, 0.0, 0.0])
|
||||
RED = torch.tensor([1.0, 0.0, 0.0])
|
||||
GREEN = torch.tensor([0.0, 1.0, 0.0])
|
||||
BLUE = torch.tensor([0.0, 0.0, 1.0])
|
||||
|
||||
COLORS_DICT = {
|
||||
"white": WHITE,
|
||||
"black": BLACK,
|
||||
"red": RED,
|
||||
"green": GREEN,
|
||||
"blue": BLUE,
|
||||
}
|
||||
|
||||
|
||||
def get_color(color: Union[str, list, ListConfig]) -> TensorType[3]:
|
||||
"""
|
||||
Args:
|
||||
color (Union[str, list, ListConfig]): Color as a string or a rgb list
|
||||
|
||||
Returns:
|
||||
TensorType[3]: Parsed color
|
||||
"""
|
||||
if isinstance(color, str):
|
||||
color = color.lower()
|
||||
if color not in COLORS_DICT:
|
||||
raise ValueError(f"{color} is not a valid preset color")
|
||||
return COLORS_DICT[color]
|
||||
if isinstance(color, (list, ListConfig)):
|
||||
if len(color) != 3:
|
||||
raise ValueError(f"Color should be 3 values (RGB) instead got {color}")
|
||||
return torch.tensor(color)
|
||||
|
||||
raise ValueError(f"Color should be an RGB list or string, instead got {type(color)}")
|
||||
@@ -0,0 +1,244 @@
|
||||
"""
|
||||
Some ray datastructures.
|
||||
"""
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
|
||||
from radiance.utils.misc import is_not_none
|
||||
|
||||
|
||||
@dataclass
|
||||
class PointSamples:
|
||||
"""Samples in space.
|
||||
|
||||
Args:
|
||||
positions (TensorType[..., 3]): XYZ locations
|
||||
directions (TensorType[..., 3]): Unit direction vector
|
||||
camera_indices (TensorType[..., 1]): Camera index
|
||||
valid_mask (TensorType[...]): Rays that are valid
|
||||
"""
|
||||
|
||||
positions: TensorType[..., 3]
|
||||
directions: TensorType[..., 3] = None
|
||||
camera_indices: TensorType[..., 1] = None
|
||||
valid_mask: TensorType[...] = None
|
||||
|
||||
def apply_masks(self) -> "PointSamples":
|
||||
"""Use valid_mask to mask samples.
|
||||
|
||||
Returns:
|
||||
PointSamples: New set of masked samples.
|
||||
"""
|
||||
if is_not_none(self.valid_mask):
|
||||
positions = self.positions[self.valid_mask]
|
||||
directions = self.directions[self.valid_mask] if is_not_none(self.directions) else self.directions
|
||||
camera_indices = (
|
||||
self.camera_indices[self.valid_mask] if is_not_none(self.camera_indices) else self.camera_indices
|
||||
)
|
||||
return PointSamples(
|
||||
positions=positions,
|
||||
directions=directions,
|
||||
camera_indices=camera_indices,
|
||||
)
|
||||
return PointSamples(
|
||||
positions=self.positions,
|
||||
directions=self.directions,
|
||||
camera_indices=self.camera_indices,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RaySamples:
|
||||
"""Samples along a ray
|
||||
|
||||
Args:
|
||||
positions (TensorType[..., 3]): XYZ locations
|
||||
directions (TensorType[..., 3]): Unit direction vector
|
||||
camera_indices (TensorType[..., 1]): Camera index
|
||||
valid_mask (TensorType[...]): Rays that are valid
|
||||
ts (TensorType[..., 1]): "time steps", distances along ray
|
||||
deltas )TensorType[..., 1]): "width" of each sample
|
||||
"""
|
||||
|
||||
positions: TensorType[..., 3]
|
||||
directions: TensorType[..., 3] = None
|
||||
camera_indices: TensorType[..., 1] = None
|
||||
valid_mask: TensorType[...] = None
|
||||
ts: TensorType[..., 1] = None
|
||||
deltas: TensorType[..., 1] = None
|
||||
|
||||
def to_point_samples(self) -> PointSamples:
|
||||
"""Convert to PointSamples instance and return."""
|
||||
# TODO: make this more interpretable
|
||||
return PointSamples(positions=self.positions, directions=self.directions, valid_mask=self.valid_mask)
|
||||
|
||||
def get_weights(self, densities: TensorType[..., "num_samples", 1]) -> TensorType[..., "num_samples"]:
|
||||
"""Return weights based on predicted densities
|
||||
|
||||
Args:
|
||||
densities (TensorType[..., "num_samples", 1]): Predicted densities for samples along ray
|
||||
|
||||
Returns:
|
||||
TensorType[..., "num_samples"]: Weights for each sample
|
||||
"""
|
||||
|
||||
delta_density = self.deltas * densities[..., 0]
|
||||
alphas = 1 - torch.exp(-delta_density)
|
||||
|
||||
# mip-nerf version of transmittance calculation:
|
||||
transmittance = torch.cumsum(delta_density[..., :-1], dim=-1)
|
||||
transmittance = torch.cat(
|
||||
[torch.zeros((*transmittance.shape[:1], 1)).to(densities.device), transmittance], axis=-1
|
||||
)
|
||||
transmittance = torch.exp(-transmittance) # [..., "num_samples"]
|
||||
|
||||
# most nerf codebases do the following:
|
||||
# transmittance = torch.cat(
|
||||
# [torch.ones((*alphas.shape[:1], 1)).to(densities.device), 1.0 - alphas + 1e-10], dim=-1
|
||||
# )
|
||||
# transmittance = torch.cumprod(transmittance, dim=-1)[..., :-1] # [..., "num_samples"]
|
||||
|
||||
weights = alphas * transmittance # [..., "num_samples"]
|
||||
|
||||
return weights
|
||||
|
||||
def set_valid_mask(self, valid_mask: TensorType[..., "num_samples"]) -> None:
|
||||
"""Sets valid mask"""
|
||||
self.valid_mask = valid_mask
|
||||
|
||||
|
||||
@dataclass
|
||||
class RayBundle:
|
||||
"""A bundle of ray parameters.
|
||||
|
||||
Args:
|
||||
origins (TensorType[..., 3]): Ray origins (XYZ)
|
||||
directions (TensorType[..., 3]): Unit ray direction vector
|
||||
camera_indices (TensorType[..., 1]): Camera indices
|
||||
nears (TensorType[..., 1]): Distance along ray to start sampling
|
||||
fars (TensorType[..., 1]): Rays Distance along ray to stop sampling
|
||||
valid_mask (TensorType[...]): Rays that are valid
|
||||
"""
|
||||
|
||||
origins: TensorType["num_rays", 3]
|
||||
directions: TensorType["num_rays", 3]
|
||||
camera_indices: Optional[TensorType["num_rays", 1]] = None
|
||||
nears: Optional[TensorType["num_rays"]] = None
|
||||
fars: Optional[TensorType["num_rays"]] = None
|
||||
valid_mask: Optional[TensorType["num_rays"]] = None
|
||||
|
||||
def move_to_device(self, device: torch.device) -> None:
|
||||
"""Move bundle data to a device.
|
||||
|
||||
Args:
|
||||
device (torch.device): Device to move tensors to.
|
||||
"""
|
||||
self.origins = self.origins.to(device)
|
||||
self.directions = self.directions.to(device)
|
||||
if not isinstance(self.camera_indices, type(None)):
|
||||
self.camera_indices = self.camera_indices.to(device)
|
||||
|
||||
def set_camera_indices(self, camera_index: int) -> None:
|
||||
"""Sets all of the the camera indices to a specific camera index.
|
||||
|
||||
Args:
|
||||
camera_index (int): Camera index.
|
||||
"""
|
||||
self.camera_indices = torch.ones_like(self.origins[..., 0]).long() * camera_index
|
||||
|
||||
def __len__(self):
|
||||
num_rays = torch.numel(self.origins) // self.origins.shape[-1]
|
||||
return num_rays
|
||||
|
||||
def sample(self, num_rays: int) -> "RayBundle":
|
||||
"""Returns a RayBundle as a subset of rays.
|
||||
|
||||
Args:
|
||||
num_rays (int): Number of rays in output RayBundle
|
||||
|
||||
Returns:
|
||||
RayBundle: RayBundle with subset of rays.
|
||||
"""
|
||||
assert num_rays <= len(self)
|
||||
indices = random.sample(range(len(self)), k=num_rays)
|
||||
return RayBundle(
|
||||
origins=self.origins[indices],
|
||||
directions=self.directions[indices],
|
||||
camera_indices=self.camera_indices[indices],
|
||||
)
|
||||
|
||||
def get_masked_ray_bundle(self, valid_mask: TensorType) -> "RayBundle":
|
||||
"""Return a masked instance of the ray bundle.
|
||||
|
||||
Args:
|
||||
valid_mask (TensorType): Mask of rays to keep.
|
||||
|
||||
Returns:
|
||||
RayBundle: RayBundle of masked rays.
|
||||
"""
|
||||
return RayBundle(
|
||||
origins=self.origins[valid_mask],
|
||||
directions=self.directions[valid_mask],
|
||||
camera_indices=self.camera_indices[valid_mask] if is_not_none(self.camera_indices) else None,
|
||||
nears=self.nears[valid_mask] if is_not_none(self.nears) else None,
|
||||
fars=self.fars[valid_mask] if is_not_none(self.fars) else None,
|
||||
valid_mask=self.valid_mask[valid_mask] if is_not_none(self.valid_mask) else None,
|
||||
)
|
||||
|
||||
def get_row_major_sliced_ray_bundle(self, start_idx: int, end_idx: int) -> "RayBundle":
|
||||
"""Flattens RayBundle and extracts chunk given start and end indicies.
|
||||
|
||||
Args:
|
||||
start_idx (int): Start index of RayBundle chunk.
|
||||
end_idx (int): End index of RayBundle chunk.
|
||||
|
||||
Returns:
|
||||
RayBundle: Flattened RayBundle with end_idx-start_idx rays.
|
||||
|
||||
"""
|
||||
camera_indices = (
|
||||
self.camera_indices.view(-1)[start_idx:end_idx] if not isinstance(self.camera_indices, type(None)) else None
|
||||
)
|
||||
return RayBundle(
|
||||
origins=self.origins.view(-1, 3)[start_idx:end_idx],
|
||||
directions=self.directions.view(-1, 3)[start_idx:end_idx],
|
||||
camera_indices=camera_indices,
|
||||
)
|
||||
|
||||
def get_ray_samples(self, ts: TensorType[..., "num_samples"]) -> RaySamples:
|
||||
"""Produces samples for each ray by projection points along the ray direction.
|
||||
|
||||
|
||||
Args:
|
||||
ts (TensorType[..., "num_samples"]): Distance from origin of sample points.
|
||||
|
||||
Returns:
|
||||
RaySamples: Samples projected along ray.
|
||||
"""
|
||||
positions = self.origins[:, None] + ts[:, :, None] * self.directions[:, None]
|
||||
directions = self.directions.unsqueeze(1).repeat(1, positions.shape[1], 1)
|
||||
valid_mask = torch.ones_like(ts, dtype=torch.bool)
|
||||
|
||||
dists = ts[..., 1:] - ts[..., :-1]
|
||||
dists = torch.cat([dists, dists[..., -1:]], -1) # [N_rays, N_samples]
|
||||
deltas = dists * torch.norm(self.directions[..., None, :], dim=-1)
|
||||
|
||||
if self.camera_indices is not None:
|
||||
camera_indices = self.camera_indices.unsqueeze(1).repeat(1, positions.shape[1])
|
||||
else:
|
||||
camera_indices = None
|
||||
|
||||
ray_samples = RaySamples(
|
||||
positions=positions,
|
||||
directions=directions,
|
||||
camera_indices=camera_indices,
|
||||
valid_mask=valid_mask,
|
||||
ts=ts,
|
||||
deltas=deltas,
|
||||
)
|
||||
|
||||
return ray_samples
|
||||
Reference in new issue
Block a user