torchtyping => jaxtyping (#1838)

* Run `ruff --fix`

* Add jaxtyping migration script

* Run migration scripts + ruff + black

* Merge remote-tracking branch 'origin/main' into brent/jaxtyping

* black

* Fix lint

* fix lint

* Cleanup and more specific types

* Fixed annotation errors

* Fix more errors

* Fix annotation

* Delete _torchtyping_to_jaxtyping.py

---------

Co-authored-by: Matt Tancik <mattt4001@gmail.com>
This commit is contained in:
Brent YiandMatt Tancik authored and GitHub committed 2023-05-16 15:57:47 -07:00
1 parent 9b6290ea43
commit 6ba2a18b2b
65 files changed
+597 -531

No files matched your search

+62 -50
View File
@@ -19,8 +19,8 @@ from enum import Enum
from typing import Dict, Literal
import torch
from torch import nn
from torchtyping import TensorType
from jaxtyping import Bool, Float
from torch import Tensor, nn
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils.math import masked_reduction, normalized_depth_scale_and_shift
@@ -44,12 +44,12 @@ class DepthLossType(Enum):
def outer(
t0_starts: TensorType[..., "num_samples_0"],
t0_ends: TensorType[..., "num_samples_0"],
t1_starts: TensorType[..., "num_samples_1"],
t1_ends: TensorType[..., "num_samples_1"],
y1: TensorType[..., "num_samples_1"],
) -> TensorType[..., "num_samples_0"]:
t0_starts: Float[Tensor, "*batch num_samples_0"],
t0_ends: Float[Tensor, "*batch num_samples_0"],
t1_starts: Float[Tensor, "*batch num_samples_1"],
t1_ends: Float[Tensor, "*batch num_samples_1"],
y1: Float[Tensor, "*batch num_samples_1"],
) -> Float[Tensor, "*batch num_samples_0"]:
"""Faster version of
https://github.com/kakaobrain/NeRF-Factory/blob/f61bb8744a5cb4820a4d968fb3bfbed777550f4a/src/model/mipnerf360/helper.py#L117
@@ -76,10 +76,10 @@ def outer(
def lossfun_outer(
t: TensorType[..., "num_samples+1"],
w: TensorType[..., "num_samples"],
t_env: TensorType[..., "num_samples+1"],
w_env: TensorType[..., "num_samples"],
t: Float[Tensor, "*batch num_samples+1"],
w: Float[Tensor, "*batch num_samples"],
t_env: Float[Tensor, "*batch num_samples+1"],
w_env: Float[Tensor, "*batch num_samples"],
):
"""
https://github.com/kakaobrain/NeRF-Factory/blob/f61bb8744a5cb4820a4d968fb3bfbed777550f4a/src/model/mipnerf360/helper.py#L136
@@ -145,9 +145,9 @@ def distortion_loss(weights_list, ray_samples_list):
def nerfstudio_distortion_loss(
ray_samples: RaySamples,
densities: TensorType["bs":..., "num_samples", 1] = None,
weights: TensorType["bs":..., "num_samples", 1] = None,
) -> TensorType["bs":..., 1]:
densities: Float[Tensor, "*bs num_samples 1"] = None,
weights: Float[Tensor, "*bs num_samples 1"] = None,
) -> Float[Tensor, "*bs 1"]:
"""Ray based distortion loss proposed in MipNeRF-360. Returns distortion Loss.
.. math::
@@ -186,9 +186,9 @@ def nerfstudio_distortion_loss(
def orientation_loss(
weights: TensorType["bs":..., "num_samples", 1],
normals: TensorType["bs":..., "num_samples", 3],
viewdirs: TensorType["bs":..., 3],
weights: Float[Tensor, "*bs num_samples 1"],
normals: Float[Tensor, "*bs num_samples 3"],
viewdirs: Float[Tensor, "*bs 3"],
):
"""Orientation loss proposed in Ref-NeRF.
Loss that encourages that all visible normals are facing towards the camera.
@@ -201,21 +201,21 @@ def orientation_loss(
def pred_normal_loss(
weights: TensorType["bs":..., "num_samples", 1],
normals: TensorType["bs":..., "num_samples", 3],
pred_normals: TensorType["bs":..., "num_samples", 3],
weights: Float[Tensor, "*bs num_samples 1"],
normals: Float[Tensor, "*bs num_samples 3"],
pred_normals: Float[Tensor, "*bs num_samples 3"],
):
"""Loss between normals calculated from density and normals from prediction network."""
return (weights[..., 0] * (1.0 - torch.sum(normals * pred_normals, dim=-1))).sum(dim=-1)
def ds_nerf_depth_loss(
weights: TensorType[..., "num_samples", 1],
termination_depth: TensorType[..., 1],
steps: TensorType[..., "num_samples", 1],
lengths: TensorType[..., "num_samples", 1],
sigma: TensorType[0],
) -> TensorType[..., 1]:
weights: Float[Tensor, "*batch num_samples 1"],
termination_depth: Float[Tensor, "*batch 1"],
steps: Float[Tensor, "*batch num_samples 1"],
lengths: Float[Tensor, "*batch num_samples 1"],
sigma: Float[Tensor, "0"],
) -> Float[Tensor, "*batch 1"]:
"""Depth loss from Depth-supervised NeRF (Deng et al., 2022).
Args:
@@ -235,12 +235,12 @@ def ds_nerf_depth_loss(
def urban_radiance_field_depth_loss(
weights: TensorType[..., "num_samples", 1],
termination_depth: TensorType[..., 1],
predicted_depth: TensorType[..., 1],
steps: TensorType[..., "num_samples", 1],
sigma: TensorType[0],
) -> TensorType[..., 1]:
weights: Float[Tensor, "*batch num_samples 1"],
termination_depth: Float[Tensor, "*batch 1"],
predicted_depth: Float[Tensor, "*batch 1"],
steps: Float[Tensor, "*batch num_samples 1"],
sigma: Float[Tensor, "0"],
) -> Float[Tensor, "*batch 1"]:
"""Lidar losses from Urban Radiance Fields (Rematas et al., 2022).
Args:
@@ -274,15 +274,15 @@ def urban_radiance_field_depth_loss(
def depth_loss(
weights: TensorType[..., "num_samples", 1],
weights: Float[Tensor, "*batch num_samples 1"],
ray_samples: RaySamples,
termination_depth: TensorType[..., 1],
predicted_depth: TensorType[..., 1],
sigma: TensorType[0],
directions_norm: TensorType[..., 1],
termination_depth: Float[Tensor, "*batch 1"],
predicted_depth: Float[Tensor, "*batch 1"],
sigma: Float[Tensor, "0"],
directions_norm: Float[Tensor, "*batch 1"],
is_euclidean: bool,
depth_loss_type: DepthLossType,
) -> TensorType[0]:
) -> Float[Tensor, "0"]:
"""Implementation of depth losses.
Args:
@@ -313,8 +313,8 @@ def depth_loss(
def monosdf_normal_loss(
normal_pred: TensorType["num_samples", 3], normal_gt: TensorType["num_samples", 3]
) -> TensorType[0]:
normal_pred: Float[Tensor, "num_samples 3"], normal_gt: Float[Tensor, "num_samples 3"]
) -> Float[Tensor, "0"]:
"""
Normal consistency loss proposed in monosdf - https://niujinshuchong.github.io/monosdf/
Enforces consistency between the volume rendered normal and the predicted monocular normal.
@@ -343,8 +343,11 @@ class MiDaSMSELoss(nn.Module):
self.mse_loss = MSELoss(reduction="none")
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
self,
prediction: Float[Tensor, "1 32 mult"],
target: Float[Tensor, "1 32 mult"],
mask: Bool[Tensor, "1 32 mult"],
) -> Float[Tensor, "0"]:
"""
Args:
prediction: predicted depth map
@@ -380,8 +383,11 @@ class GradientLoss(nn.Module):
self.__scales = scales
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
self,
prediction: Float[Tensor, "1 32 mult"],
target: Float[Tensor, "1 32 mult"],
mask: Bool[Tensor, "1 32 mult"],
) -> Float[Tensor, "0"]:
"""
Args:
prediction: predicted depth map
@@ -405,8 +411,11 @@ class GradientLoss(nn.Module):
return total
def gradient_loss(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
self,
prediction: Float[Tensor, "1 32 mult"],
target: Float[Tensor, "1 32 mult"],
mask: Bool[Tensor, "1 32 mult"],
) -> Float[Tensor, "0"]:
"""
multiscale, scale-invariant gradient matching term to the disparity space.
This term biases discontinuities to be sharp and to coincide with discontinuities in the ground truth
@@ -458,8 +467,11 @@ class ScaleAndShiftInvariantLoss(nn.Module):
self.__prediction_ssi = None
def forward(
self, prediction: TensorType[1, 32, "mult"], target: TensorType[1, 32, "mult"], mask: TensorType[1, 32, "mult"]
) -> TensorType[0]:
self,
prediction: Float[Tensor, "1 32 mult"],
target: Float[Tensor, "1 32 mult"],
mask: Bool[Tensor, "1 32 mult"],
) -> Float[Tensor, "0"]:
"""
Args:
prediction: predicted depth map (unnormalized)
@@ -487,7 +499,7 @@ class ScaleAndShiftInvariantLoss(nn.Module):
prediction_ssi = property(__get_prediction_ssi)
def tv_loss(grids: TensorType["grids", "feature_dim", "row", "column"]) -> TensorType[()]:
def tv_loss(grids: Float[Tensor, "grids feature_dim row column"]) -> Float[Tensor, ""]:
"""
https://github.com/apchenstu/TensoRF/blob/4ec894dc1341a2201fe13ae428631b58458f105d/utils.py#L139
@@ -15,8 +15,8 @@
"""
Ray generator.
"""
from torch import nn
from torchtyping import TensorType
from jaxtyping import Int
from torch import Tensor, nn
from nerfstudio.cameras.camera_optimizers import CameraOptimizer
from nerfstudio.cameras.cameras import Cameras
@@ -38,7 +38,7 @@ class RayGenerator(nn.Module):
self.pose_optimizer = pose_optimizer
self.register_buffer("image_coords", cameras.get_image_coords(), persistent=False)
def forward(self, ray_indices: TensorType["num_rays", 3]) -> RayBundle:
def forward(self, ray_indices: Int[Tensor, "num_rays 3"]) -> RayBundle:
"""Index into the cameras to generate the rays.
Args:
+11 -8
View File
@@ -20,9 +20,9 @@ from abc import abstractmethod
from typing import Callable, List, Optional, Tuple, Union
import torch
from jaxtyping import Float
from nerfacc import OccGridEstimator
from torch import nn
from torchtyping import TensorType
from torch import Tensor, nn
from nerfstudio.cameras.rays import Frustums, RayBundle, RaySamples
@@ -111,7 +111,10 @@ class SpacedSampler(Sampler):
bins = bin_lower + (bin_upper - bin_lower) * t_rand
s_near, s_far = (self.spacing_fn(x) for x in (ray_bundle.nears, ray_bundle.fars))
spacing_to_euclidean_fn = lambda x: self.spacing_fn_inv(x * s_far + (1 - x) * s_near)
def spacing_to_euclidean_fn(x):
return self.spacing_fn_inv(x * s_far + (1 - x) * s_near)
euclidean_bins = spacing_to_euclidean_fn(bins) # [num_rays, num_samples+1]
ray_samples = ray_bundle.get_ray_samples(
@@ -274,7 +277,7 @@ class PDFSampler(Sampler):
self,
ray_bundle: Optional[RayBundle] = None,
ray_samples: Optional[RaySamples] = None,
weights: TensorType[..., "num_samples", 1] = None,
weights: Float[Tensor, "*batch num_samples 1"] = None,
num_samples: Optional[int] = None,
eps: float = 1e-5,
) -> RaySamples:
@@ -382,7 +385,7 @@ class VolumetricSampler(Sampler):
def __init__(
self,
occupancy_grid: OccGridEstimator,
density_fn: Optional[Callable[[TensorType[..., 3]], TensorType[..., 1]]] = None,
density_fn: Optional[Callable[[Float[Tensor, "*batch 3"]], Float[Tensor, "*batch 1"]]] = None,
) -> None:
super().__init__()
assert occupancy_grid is not None
@@ -429,7 +432,7 @@ class VolumetricSampler(Sampler):
far_plane: Optional[float] = None,
alpha_thre: float = 0.01,
cone_angle: float = 0.0,
) -> Tuple[RaySamples, TensorType["total_samples",]]:
) -> Tuple[RaySamples, Float[Tensor, "total_samples "]]:
"""Generate ray samples in a bounding box.
Args:
@@ -683,8 +686,8 @@ class NeuSSampler(Sampler):
@staticmethod
def rendering_sdf_with_fixed_inv_s(
ray_samples: RaySamples, sdf: TensorType["num_samples", -1], inv_s: int
) -> TensorType["num_samples", -1]:
ray_samples: RaySamples, sdf: Float[Tensor, "num_samples 1"], inv_s: int
) -> Float[Tensor, "num_samples 1"]:
"""
rendering given a fixed inv_s as NeuS
+35 -33
View File
@@ -32,18 +32,18 @@ from typing import Generator, Literal, Optional, Union
import nerfacc
import torch
from torch import nn
from torchtyping import TensorType
from jaxtyping import Float, Int
from torch import Tensor, nn
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils import colors
from nerfstudio.utils.math import components_from_spherical_harmonics, safe_normalize
BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None
BACKGROUND_COLOR_OVERRIDE: Optional[Float[Tensor, "3"]] = None
@contextlib.contextmanager
def background_color_override_context(mode: TensorType[3]) -> Generator[None, None, None]:
def background_color_override_context(mode: Float[Tensor, "3"]) -> Generator[None, None, None]:
"""Context manager for setting background mode."""
global BACKGROUND_COLOR_OVERRIDE # pylint: disable=global-statement
old_background_color = BACKGROUND_COLOR_OVERRIDE
@@ -61,19 +61,21 @@ class RGBRenderer(nn.Module):
background_color: Background color as RGB. Uses random colors if None.
"""
def __init__(self, background_color: Union[Literal["random", "last_sample"], TensorType[3]] = "random") -> None:
def __init__(
self, background_color: Union[Literal["random", "last_sample"], Float[Tensor, "3"]] = "random"
) -> None:
super().__init__()
self.background_color = background_color
@classmethod
def combine_rgb(
cls,
rgb: TensorType["bs":..., "num_samples", 3],
weights: TensorType["bs":..., "num_samples", 1],
background_color: Union[Literal["random", "white", "black", "last_sample"], TensorType[3]] = "random",
ray_indices: Optional[TensorType["num_samples"]] = None,
rgb: Float[Tensor, "*bs num_samples 3"],
weights: Float[Tensor, "*bs num_samples 1"],
background_color: Union[Literal["random", "white", "black", "last_sample"], Float[Tensor, "3"]] = "random",
ray_indices: Optional[Int[Tensor, "num_samples"]] = None,
num_rays: Optional[int] = None,
) -> TensorType["bs":..., 3]:
) -> Float[Tensor, "*bs 3"]:
"""Composite samples along ray and render color image
Args:
@@ -116,11 +118,11 @@ class RGBRenderer(nn.Module):
def forward(
self,
rgb: TensorType["bs":..., "num_samples", 3],
weights: TensorType["bs":..., "num_samples", 1],
ray_indices: Optional[TensorType["num_samples"]] = None,
rgb: Float[Tensor, "*bs num_samples 3"],
weights: Float[Tensor, "*bs num_samples 1"],
ray_indices: Optional[Int[Tensor, "num_samples"]] = None,
num_rays: Optional[int] = None,
) -> TensorType["bs":..., 3]:
) -> Float[Tensor, "*bs 3"]:
"""Composite samples along ray and render color image
Args:
@@ -153,7 +155,7 @@ class SHRenderer(nn.Module):
def __init__(
self,
background_color: Union[Literal["random", "last_sample"], TensorType[3]] = "random",
background_color: Union[Literal["random", "last_sample"], Float[Tensor, "3"]] = "random",
activation: Optional[nn.Module] = nn.Sigmoid(),
) -> None:
super().__init__()
@@ -162,10 +164,10 @@ class SHRenderer(nn.Module):
def forward(
self,
sh: TensorType[..., "num_samples", "coeffs"],
directions: TensorType[..., "num_samples", 3],
weights: TensorType[..., "num_samples", 1],
) -> TensorType[..., 3]:
sh: Float[Tensor, "*batch num_samples coeffs"],
directions: Float[Tensor, "*batch num_samples 3"],
weights: Float[Tensor, "*batch num_samples 1"],
) -> Float[Tensor, "*batch 3"]:
"""Composite samples along ray and render color image
Args:
@@ -203,10 +205,10 @@ class AccumulationRenderer(nn.Module):
@classmethod
def forward(
cls,
weights: TensorType["bs":..., "num_samples", 1],
ray_indices: Optional[TensorType["num_samples"]] = None,
weights: Float[Tensor, "*bs num_samples 1"],
ray_indices: Optional[Int[Tensor, "num_samples"]] = None,
num_rays: Optional[int] = None,
) -> TensorType["bs":..., 1]:
) -> Float[Tensor, "*bs 1"]:
"""Composite samples along ray and calculate accumulation.
Args:
@@ -245,11 +247,11 @@ class DepthRenderer(nn.Module):
def forward(
self,
weights: TensorType[..., "num_samples", 1],
weights: Float[Tensor, "*batch num_samples 1"],
ray_samples: RaySamples,
ray_indices: Optional[TensorType["num_samples"]] = None,
ray_indices: Optional[Int[Tensor, "num_samples"]] = None,
num_rays: Optional[int] = None,
) -> TensorType[..., 1]:
) -> Float[Tensor, "*batch 1"]:
"""Composite samples along ray and calculate depths.
Args:
@@ -301,8 +303,8 @@ class UncertaintyRenderer(nn.Module):
@classmethod
def forward(
cls, betas: TensorType["bs":..., "num_samples", 1], weights: TensorType["bs":..., "num_samples", 1]
) -> TensorType["bs":..., 1]:
cls, betas: Float[Tensor, "*bs num_samples 1"], weights: Float[Tensor, "*bs num_samples 1"]
) -> Float[Tensor, "*bs 1"]:
"""Calculate uncertainty along the ray.
Args:
@@ -322,9 +324,9 @@ class SemanticRenderer(nn.Module):
@classmethod
def forward(
cls,
semantics: TensorType["bs":..., "num_samples", "num_classes"],
weights: TensorType["bs":..., "num_samples", 1],
) -> TensorType["bs":..., "num_classes"]:
semantics: Float[Tensor, "*bs num_samples num_classes"],
weights: Float[Tensor, "*bs num_samples 1"],
) -> Float[Tensor, "*bs num_classes"]:
"""Calculate semantics along the ray."""
sem = torch.sum(weights * semantics, dim=-2)
return sem
@@ -336,10 +338,10 @@ class NormalsRenderer(nn.Module):
@classmethod
def forward(
cls,
normals: TensorType["bs":..., "num_samples", 3],
weights: TensorType["bs":..., "num_samples", 1],
normals: Float[Tensor, "*bs num_samples 3"],
weights: Float[Tensor, "*bs num_samples 1"],
normalize: bool = True,
) -> TensorType["bs":..., 3]:
) -> Float[Tensor, "*bs 3"]:
"""Calculate normals along the ray.
Args:
@@ -19,8 +19,8 @@ Scene Colliders
from __future__ import annotations
import torch
from torch import nn
from torchtyping import TensorType
from jaxtyping import Float
from torch import Tensor, nn
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.data.scene_box import SceneBox
@@ -57,7 +57,7 @@ class AABBBoxCollider(SceneCollider):
self.near_plane = near_plane
def _intersect_with_aabb(
self, rays_o: TensorType["num_rays", 3], rays_d: TensorType["num_rays", 3], aabb: TensorType[2, 3]
self, rays_o: Float[Tensor, "num_rays 3"], rays_d: Float[Tensor, "num_rays 3"], aabb: Float[Tensor, "2 3"]
):
"""Returns collection of valid rays within a specified near/far bounding box along with a mask
specifying which rays are valid
+7 -7
View File
@@ -15,8 +15,8 @@
"""Shaders for rendering."""
from typing import Optional
from torch import nn
from torchtyping import TensorType
from jaxtyping import Float
from torch import Tensor, nn
class LambertianShader(nn.Module):
@@ -25,9 +25,9 @@ class LambertianShader(nn.Module):
@classmethod
def forward(
cls,
rgb: TensorType["bs":..., 3],
normals: TensorType["bs":..., 3],
light_direction: TensorType["bs":..., 3],
rgb: Float[Tensor, "*bs 3"],
normals: Float[Tensor, "*bs 3"],
light_direction: Float[Tensor, "*bs 3"],
shading_weight: float = 1.0,
detach_normals=True,
):
@@ -59,8 +59,8 @@ class NormalsShader(nn.Module):
@classmethod
def forward(
cls,
normals: TensorType["bs":..., 3],
weights: Optional[TensorType["bs":..., 1]] = None,
normals: Float[Tensor, "*bs 3"],
weights: Optional[Float[Tensor, "*bs 1"]] = None,
):
"""Applies a rainbow colormap to the normals.