Files
nerfstudio/nerfstudio/cameras/camera_optimizers.py
T
2023-06-20 05:45:58 +00:00

122 lines
4.3 KiB
Python

# Copyright 2022 the Regents of the University of California, Nerfstudio Team and contributors. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Pose and Intrinsics Optimizers
"""
from __future__ import annotations
import functools
from dataclasses import dataclass, field
from typing import Literal, Type, Union
import torch
from jaxtyping import Float, Int
from torch import Tensor, nn
from typing_extensions import assert_never
from nerfstudio.cameras.lie_groups import exp_map_SE3, exp_map_SO3xR3
from nerfstudio.cameras.rays import RayBundle
from nerfstudio.configs.base_config import InstantiateConfig
from nerfstudio.utils import poses as pose_utils
@dataclass
class CameraOptimizerConfig(InstantiateConfig):
"""Configuration of optimization for camera poses."""
_target: Type = field(default_factory=lambda: CameraOptimizer)
mode: Literal["off", "SO3xR3", "SE3"] = "off"
"""Pose optimization strategy to use. If enabled, we recommend SO3xR3."""
trans_l2_penalty: float = 1e-2
"""L2 penalty on translation parameters."""
rot_l2_penalty: float = 1e-3
"""L2 penalty on rotation parameters."""
class CameraOptimizer(nn.Module):
"""Layer that modifies camera poses to be optimized as well as the field during training."""
config: CameraOptimizerConfig
def __init__(
self,
config: CameraOptimizerConfig,
num_cameras: int,
device: Union[torch.device, str],
**kwargs,
) -> None:
super().__init__()
self.config = config
self.num_cameras = num_cameras
self.device = device
# Initialize learnable parameters.
if self.config.mode == "off":
pass
elif self.config.mode in ("SO3xR3", "SE3"):
self.pose_adjustment = torch.nn.Parameter(torch.zeros((num_cameras, 6), device=device))
else:
assert_never(self.config.mode)
def forward(
self,
indices: Int[Tensor, "num_cameras"],
) -> Float[Tensor, "num_cameras 3 4"]:
"""Indexing into camera adjustments.
Args:
indices: indices of Cameras to optimize.
Returns:
Transformation matrices from optimized camera coordinates
to given camera coordinates.
"""
outputs = []
# Apply learned transformation delta.
if self.config.mode == "off":
pass
elif self.config.mode == "SO3xR3":
outputs.append(exp_map_SO3xR3(self.pose_adjustment[indices, :]))
elif self.config.mode == "SE3":
outputs.append(exp_map_SE3(self.pose_adjustment[indices, :]))
else:
assert_never(self.config.mode)
# Return: identity if no transforms are needed, otherwise multiply transforms together.
if len(outputs) == 0:
# Note that using repeat() instead of tile() here would result in unnecessary copies.
return torch.eye(4, device=self.device)[None, :3, :4].tile(indices.shape[0], 1, 1)
return functools.reduce(pose_utils.multiply, outputs)
def apply_to_raybundle(self, raybundle: RayBundle) -> None:
"""
Apply the pose correction to the raybundle
"""
correction_matrices = self(raybundle.camera_indices.squeeze())
raybundle.origins = raybundle.origins + correction_matrices[:, :3, 3]
raybundle.directions = torch.bmm(correction_matrices[:, :3, :3], raybundle.directions[..., None]).squeeze()
def get_loss_dict(self, loss_dict: dict) -> dict:
"""
Add a regularizer
"""
loss_dict["camera_opt_regularizer"] = (
self.pose_adjustment[:, :3].norm(dim=-1).mean() * self.config.trans_l2_penalty
+ self.pose_adjustment[:, 3:].norm(dim=-1).mean() * self.config.rot_l2_penalty
)