mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 22:56:03 +08:00
torchtyping => jaxtyping (#1838)
* Run `ruff --fix` * Add jaxtyping migration script * Run migration scripts + ruff + black * Merge remote-tracking branch 'origin/main' into brent/jaxtyping * black * Fix lint * fix lint * Cleanup and more specific types * Fixed annotation errors * Fix more errors * Fix annotation * Delete _torchtyping_to_jaxtyping.py --------- Co-authored-by: Matt Tancik <mattt4001@gmail.com>
This commit is contained in:
1 parent
9b6290ea43
commit
6ba2a18b2b
65 files changed
+597
-531
No files matched your search
@@ -19,7 +19,8 @@ from typing import Literal, Optional
|
||||
|
||||
import matplotlib
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
from jaxtyping import Bool, Float
|
||||
from torch import Tensor
|
||||
|
||||
from nerfstudio.utils import colors
|
||||
|
||||
@@ -43,10 +44,10 @@ class ColormapOptions:
|
||||
|
||||
|
||||
def apply_colormap(
|
||||
image: TensorType["bs":..., 1],
|
||||
image: Float[Tensor, "*bs channels"],
|
||||
colormap_options: ColormapOptions = ColormapOptions(),
|
||||
eps: float = 1e-9,
|
||||
) -> TensorType["bs":..., "rgb":3]:
|
||||
) -> Float[Tensor, "*bs rgb=3"]:
|
||||
"""
|
||||
Applies a colormap to a tensor image.
|
||||
If single channel, applies a colormap to the image.
|
||||
@@ -89,9 +90,7 @@ def apply_colormap(
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def apply_float_colormap(
|
||||
image: TensorType["bs":..., 1], colormap: Colormaps = "viridis"
|
||||
) -> TensorType["bs":..., "rgb":3]:
|
||||
def apply_float_colormap(image: Float[Tensor, "*bs 1"], colormap: Colormaps = "viridis") -> Float[Tensor, "*bs rgb=3"]:
|
||||
"""Convert single channel to a color image.
|
||||
|
||||
Args:
|
||||
@@ -99,7 +98,7 @@ def apply_float_colormap(
|
||||
colormap: Colormap for image.
|
||||
|
||||
Returns:
|
||||
TensorType: Colored image with colors in [0, 1]
|
||||
Tensor: Colored image with colors in [0, 1]
|
||||
"""
|
||||
if colormap == "default":
|
||||
colormap = "turbo"
|
||||
@@ -116,12 +115,12 @@ def apply_float_colormap(
|
||||
|
||||
|
||||
def apply_depth_colormap(
|
||||
depth: TensorType["bs":..., 1],
|
||||
accumulation: Optional[TensorType["bs":..., 1]] = None,
|
||||
depth: Float[Tensor, "*bs 1"],
|
||||
accumulation: Optional[Float[Tensor, "*bs 1"]] = None,
|
||||
near_plane: Optional[float] = None,
|
||||
far_plane: Optional[float] = None,
|
||||
colormap_options: ColormapOptions = ColormapOptions(),
|
||||
) -> TensorType["bs":..., "rgb":3]:
|
||||
) -> Float[Tensor, "*bs rgb=3"]:
|
||||
"""Converts a depth image to color for easier analysis.
|
||||
|
||||
Args:
|
||||
@@ -151,10 +150,10 @@ def apply_depth_colormap(
|
||||
|
||||
|
||||
def apply_boolean_colormap(
|
||||
image: TensorType["bs":..., 1, bool],
|
||||
true_color: TensorType["bs":..., "rgb":3] = colors.WHITE,
|
||||
false_color: TensorType["bs":..., "rgb":3] = colors.BLACK,
|
||||
) -> TensorType["bs":..., "rgb":3]:
|
||||
image: Bool[Tensor, "*bs 1"],
|
||||
true_color: Float[Tensor, "*bs rgb=3"] = colors.WHITE,
|
||||
false_color: Float[Tensor, "*bs rgb=3"] = colors.BLACK,
|
||||
) -> Float[Tensor, "*bs rgb=3"]:
|
||||
"""Converts a depth image to color for easier analysis.
|
||||
|
||||
Args:
|
||||
@@ -172,7 +171,7 @@ def apply_boolean_colormap(
|
||||
return colored_image
|
||||
|
||||
|
||||
def apply_pca_colormap(image: TensorType["bs":..., "dim"]) -> TensorType["bs":..., "rgb":3]:
|
||||
def apply_pca_colormap(image: Float[Tensor, "*bs dim"]) -> Float[Tensor, "*bs rgb=3"]:
|
||||
"""Convert feature image to 3-channel RGB via PCA. The first three principle
|
||||
components are used for the color channels, with outlier rejection per-channel
|
||||
|
||||
@@ -180,7 +179,7 @@ def apply_pca_colormap(image: TensorType["bs":..., "dim"]) -> TensorType["bs":..
|
||||
image: image of arbitrary vectors
|
||||
|
||||
Returns:
|
||||
TensorType: Colored image
|
||||
Tensor: Colored image
|
||||
"""
|
||||
original_shape = image.shape
|
||||
image = image.view(-1, image.shape[-1])
|
||||
|
||||
@@ -16,7 +16,8 @@
|
||||
from typing import Union
|
||||
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
from jaxtyping import Float
|
||||
from torch import Tensor
|
||||
|
||||
WHITE = torch.tensor([1.0, 1.0, 1.0])
|
||||
BLACK = torch.tensor([0.0, 0.0, 0.0])
|
||||
@@ -33,13 +34,13 @@ COLORS_DICT = {
|
||||
}
|
||||
|
||||
|
||||
def get_color(color: Union[str, list]) -> TensorType[3]:
|
||||
def get_color(color: Union[str, list]) -> Float[Tensor, "3"]:
|
||||
"""
|
||||
Args:
|
||||
color (Union[str, list]): Color as a string or a rgb list
|
||||
Color as a string or a rgb list
|
||||
|
||||
Returns:
|
||||
TensorType[3]: Parsed color
|
||||
Parsed color
|
||||
"""
|
||||
if isinstance(color, str):
|
||||
color = color.lower()
|
||||
|
||||
+27
-22
@@ -18,10 +18,13 @@ from dataclasses import dataclass
|
||||
from typing import Literal, Tuple
|
||||
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
from jaxtyping import Bool, Float
|
||||
from torch import Tensor
|
||||
|
||||
|
||||
def components_from_spherical_harmonics(levels: int, directions: TensorType[..., 3]) -> TensorType[..., "components"]:
|
||||
def components_from_spherical_harmonics(
|
||||
levels: int, directions: Float[Tensor, "*batch 3"]
|
||||
) -> Float[Tensor, "*batch components"]:
|
||||
"""
|
||||
Returns value for each component of spherical harmonics.
|
||||
|
||||
@@ -94,15 +97,15 @@ class Gaussians:
|
||||
cov: Covariance of multivariate Gaussian.
|
||||
"""
|
||||
|
||||
mean: TensorType[..., "dim"]
|
||||
cov: TensorType[..., "dim", "dim"]
|
||||
mean: Float[Tensor, "*batch dim"]
|
||||
cov: Float[Tensor, "*batch dim dim"]
|
||||
|
||||
|
||||
def compute_3d_gaussian(
|
||||
directions: TensorType[..., 3],
|
||||
means: TensorType[..., 3],
|
||||
dir_variance: TensorType[..., 1],
|
||||
radius_variance: TensorType[..., 1],
|
||||
directions: Float[Tensor, "*batch 3"],
|
||||
means: Float[Tensor, "*batch 3"],
|
||||
dir_variance: Float[Tensor, "*batch 1"],
|
||||
radius_variance: Float[Tensor, "*batch 1"],
|
||||
) -> Gaussians:
|
||||
"""Compute gaussian along ray.
|
||||
|
||||
@@ -127,11 +130,11 @@ def compute_3d_gaussian(
|
||||
|
||||
|
||||
def cylinder_to_gaussian(
|
||||
origins: TensorType[..., 3],
|
||||
directions: TensorType[..., 3],
|
||||
starts: TensorType[..., 1],
|
||||
ends: TensorType[..., 1],
|
||||
radius: TensorType[..., 1],
|
||||
origins: Float[Tensor, "*batch 3"],
|
||||
directions: Float[Tensor, "*batch 3"],
|
||||
starts: Float[Tensor, "*batch 1"],
|
||||
ends: Float[Tensor, "*batch 1"],
|
||||
radius: Float[Tensor, "*batch 1"],
|
||||
) -> Gaussians:
|
||||
"""Approximates cylinders with a Gaussian distributions.
|
||||
|
||||
@@ -152,11 +155,11 @@ def cylinder_to_gaussian(
|
||||
|
||||
|
||||
def conical_frustum_to_gaussian(
|
||||
origins: TensorType[..., 3],
|
||||
directions: TensorType[..., 3],
|
||||
starts: TensorType[..., 1],
|
||||
ends: TensorType[..., 1],
|
||||
radius: TensorType[..., 1],
|
||||
origins: Float[Tensor, "*batch 3"],
|
||||
directions: Float[Tensor, "*batch 3"],
|
||||
starts: Float[Tensor, "*batch 1"],
|
||||
ends: Float[Tensor, "*batch 1"],
|
||||
radius: Float[Tensor, "*batch 1"],
|
||||
) -> Gaussians:
|
||||
"""Approximates conical frustums with a Gaussian distributions.
|
||||
|
||||
@@ -236,9 +239,9 @@ def intersect_aabb(
|
||||
|
||||
|
||||
def safe_normalize(
|
||||
vectors: TensorType["batch_dim":..., "N"],
|
||||
vectors: Float[Tensor, "*batch_dim N"],
|
||||
eps: float = 1e-10,
|
||||
) -> TensorType["batch_dim":..., "N"]:
|
||||
) -> Float[Tensor, "*batch_dim N"]:
|
||||
"""Normalizes vectors.
|
||||
|
||||
Args:
|
||||
@@ -252,7 +255,9 @@ def safe_normalize(
|
||||
|
||||
|
||||
def masked_reduction(
|
||||
input_tensor: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"], reduction_type: Literal["image", "batch"]
|
||||
input_tensor: Float[Tensor, "1 32 mult"],
|
||||
mask: Bool[Tensor, "1 32 mult"],
|
||||
reduction_type: Literal["image", "batch"],
|
||||
):
|
||||
"""
|
||||
Whether to consolidate the input_tensor across the batch or across the image
|
||||
@@ -277,7 +282,7 @@ def masked_reduction(
|
||||
|
||||
|
||||
def normalized_depth_scale_and_shift(
|
||||
prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
|
||||
prediction: Float[Tensor, "1 32 mult"], target: Float[Tensor, "1 32 mult"], mask: Bool[Tensor, "1 32 mult"]
|
||||
):
|
||||
"""
|
||||
More info here: https://arxiv.org/pdf/2206.00665.pdf supplementary section A2 Depth Consistency Loss
|
||||
|
||||
@@ -27,8 +27,9 @@ import numpy as np
|
||||
import plotly.graph_objects as go
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from jaxtyping import Float
|
||||
from plotly import express as ex
|
||||
from torchtyping import TensorType
|
||||
from torch import Tensor
|
||||
|
||||
from nerfstudio.cameras.cameras import Cameras
|
||||
from nerfstudio.cameras.rays import Frustums, RayBundle
|
||||
@@ -49,7 +50,7 @@ def color_str(color):
|
||||
|
||||
|
||||
def get_line_segments_from_lines(
|
||||
lines: TensorType["num_rays", 2, 3],
|
||||
lines: Float[Tensor, "num_rays 2 3"],
|
||||
color: str = color_str((1, 0, 0)),
|
||||
marker_color: str = color_str((1, 0, 0)),
|
||||
colors: Optional[List[str]] = None,
|
||||
@@ -99,7 +100,7 @@ def get_line_segments_from_lines(
|
||||
return data
|
||||
|
||||
|
||||
def vis_dataset(camera_origins: TensorType["num_cameras", 3], ray_bundle: RayBundle) -> go.FigureWidget: # type: ignore
|
||||
def vis_dataset(camera_origins: Float[Tensor, "num_cameras 3"], ray_bundle: RayBundle) -> go.FigureWidget:
|
||||
"""Visualize a dataset with plotly using our cameras and generated rays.
|
||||
|
||||
Args:
|
||||
@@ -165,7 +166,7 @@ def get_random_color(colormap: Optional[List[str]] = None, idx: Optional[int] =
|
||||
|
||||
|
||||
def get_sphere(
|
||||
radius: float, center: TensorType[3] = None, color: str = "black", opacity: float = 1.0, resolution: int = 32
|
||||
radius: float, center: Float[Tensor, "3"] = None, color: str = "black", opacity: float = 1.0, resolution: int = 32
|
||||
) -> go.Mesh3d: # type: ignore
|
||||
"""Returns a sphere object for plotting with plotly.
|
||||
|
||||
@@ -206,7 +207,7 @@ def get_sphere(
|
||||
|
||||
def get_cube(
|
||||
side_length: Union[float, torch.Tensor],
|
||||
center: TensorType[3] = None,
|
||||
center: Float[Tensor, "3"] = None,
|
||||
color: str = "black",
|
||||
opacity: float = 1.0,
|
||||
) -> go.Mesh3d: # type: ignore
|
||||
@@ -253,8 +254,8 @@ def get_cube(
|
||||
|
||||
|
||||
def get_gaussian_ellipsiod(
|
||||
mean: TensorType[3],
|
||||
cov: TensorType[3, 3],
|
||||
mean: Float[Tensor, "3"],
|
||||
cov: Float[Tensor, "3 3"],
|
||||
n_std: int = 2,
|
||||
color="lightblue",
|
||||
opacity: float = 0.5,
|
||||
|
||||
@@ -17,10 +17,11 @@ Common 3D pose methods
|
||||
"""
|
||||
|
||||
import torch
|
||||
from torchtyping import TensorType
|
||||
from jaxtyping import Float
|
||||
from torch import Tensor
|
||||
|
||||
|
||||
def to4x4(pose: TensorType[..., 3, 4]) -> TensorType[..., 4, 4]:
|
||||
def to4x4(pose: Float[Tensor, "*batch 3 4"]) -> Float[Tensor, "*batch 4 4"]:
|
||||
"""Convert 3x4 pose matrices to a 4x4 with the addition of a homogeneous coordinate.
|
||||
|
||||
Args:
|
||||
@@ -34,7 +35,7 @@ def to4x4(pose: TensorType[..., 3, 4]) -> TensorType[..., 4, 4]:
|
||||
return torch.cat([pose, constants], dim=-2)
|
||||
|
||||
|
||||
def inverse(pose: TensorType[..., 3, 4]) -> TensorType[..., 3, 4]:
|
||||
def inverse(pose: Float[Tensor, "*batch 3 4"]) -> Float[Tensor, "*batch 3 4"]:
|
||||
"""Invert provided pose matrix.
|
||||
|
||||
Args:
|
||||
@@ -50,7 +51,7 @@ def inverse(pose: TensorType[..., 3, 4]) -> TensorType[..., 3, 4]:
|
||||
return torch.cat([R_inverse, t_inverse], dim=-1)
|
||||
|
||||
|
||||
def multiply(pose_a: TensorType[..., 3, 4], pose_b: TensorType[..., 3, 4]) -> TensorType[..., 3, 4]:
|
||||
def multiply(pose_a: Float[Tensor, "*batch 3 4"], pose_b: Float[Tensor, "*batch 3 4"]) -> Float[Tensor, "*batch 3 4"]:
|
||||
"""Multiply two pose matrices, A @ B.
|
||||
|
||||
Args:
|
||||
@@ -67,7 +68,7 @@ def multiply(pose_a: TensorType[..., 3, 4], pose_b: TensorType[..., 3, 4]) -> Te
|
||||
return torch.cat([R, t], dim=-1)
|
||||
|
||||
|
||||
def normalize(poses: TensorType[..., 3, 4]) -> TensorType[..., 3, 4]:
|
||||
def normalize(poses: Float[Tensor, "*batch 3 4"]) -> Float[Tensor, "*batch 3 4"]:
|
||||
"""Normalize the XYZs of poses to fit within a unit cube ([-1, 1]). Note: This operation is not in-place.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -152,8 +152,12 @@ class TensorDataclass:
|
||||
if isinstance(indices, (int, slice, type(Ellipsis))):
|
||||
indices = (indices,)
|
||||
assert isinstance(indices, tuple)
|
||||
tensor_fn = lambda x: x[indices + (slice(None),)]
|
||||
dataclass_fn = lambda x: x[indices]
|
||||
|
||||
def tensor_fn(x):
|
||||
return x[indices + (slice(None),)]
|
||||
|
||||
def dataclass_fn(x):
|
||||
return x[indices]
|
||||
|
||||
def custom_tensor_dims_fn(k, v):
|
||||
custom_dims = self._field_custom_dimensions[k] # pylint: disable=unsubscriptable-object
|
||||
@@ -207,8 +211,12 @@ class TensorDataclass:
|
||||
"""
|
||||
if isinstance(shape, int):
|
||||
shape = (shape,)
|
||||
tensor_fn = lambda x: x.reshape((*shape, x.shape[-1]))
|
||||
dataclass_fn = lambda x: x.reshape(shape)
|
||||
|
||||
def tensor_fn(x):
|
||||
return x.reshape((*shape, x.shape[-1]))
|
||||
|
||||
def dataclass_fn(x):
|
||||
return x.reshape(shape)
|
||||
|
||||
def custom_tensor_dims_fn(k, v):
|
||||
custom_dims = self._field_custom_dimensions[k] # pylint: disable=unsubscriptable-object
|
||||
|
||||
@@ -26,15 +26,21 @@ from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
import torch
|
||||
import wandb
|
||||
from jaxtyping import Float
|
||||
from torch import Tensor
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
from torchtyping import TensorType
|
||||
|
||||
from nerfstudio.configs import base_config as cfg
|
||||
from nerfstudio.utils.decorators import check_main_thread, decorate_all
|
||||
from nerfstudio.utils.printing import human_format
|
||||
from nerfstudio.utils.rich_utils import CONSOLE
|
||||
|
||||
to8b = lambda x: (255 * torch.clamp(x, min=0, max=1)).to(torch.uint8)
|
||||
|
||||
def to8b(x):
|
||||
"""Converts a torch tensor to 8 bit"""
|
||||
return (255 * torch.clamp(x, min=0, max=1)).to(torch.uint8)
|
||||
|
||||
|
||||
EVENT_WRITERS = []
|
||||
EVENT_STORAGE = []
|
||||
GLOBAL_BUFFER = {}
|
||||
@@ -63,7 +69,7 @@ class EventType(enum.Enum):
|
||||
|
||||
|
||||
@check_main_thread
|
||||
def put_image(name, image: TensorType["H", "W", "C"], step: int):
|
||||
def put_image(name, image: Float[Tensor, "H W C"], step: int):
|
||||
"""Setter function to place images into the queue to be written out
|
||||
|
||||
Args:
|
||||
@@ -227,7 +233,7 @@ class Writer:
|
||||
"""Writer class"""
|
||||
|
||||
@abstractmethod
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
def write_image(self, name: str, image: Float[Tensor, "H W C"], step: int) -> None:
|
||||
"""method to write out image
|
||||
|
||||
Args:
|
||||
@@ -301,7 +307,7 @@ class WandbWriter(Writer):
|
||||
reinit=True,
|
||||
)
|
||||
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
def write_image(self, name: str, image: Float[Tensor, "H W C"], step: int) -> None:
|
||||
image = torch.permute(image, (2, 0, 1))
|
||||
wandb.log({name: wandb.Image(image)}, step=step)
|
||||
|
||||
@@ -326,7 +332,7 @@ class TensorboardWriter(Writer):
|
||||
def __init__(self, log_dir: Path):
|
||||
self.tb_writer = SummaryWriter(log_dir=log_dir)
|
||||
|
||||
def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None:
|
||||
def write_image(self, name: str, image: Float[Tensor, "H W C"], step: int) -> None:
|
||||
image = to8b(image)
|
||||
self.tb_writer.add_image(name, image, step, dataformats="HWC")
|
||||
|
||||
|
||||
Reference in new issue
Block a user