Render Omni-directional stereo video (360 3D VR) (#1986)

* Render Omni-directional stereo (360 3D VR)

Adding the omni-directional stereo camera model to render 3D 360 video and images for VR. This includes the camera models for the left and right eyes and additions to the render script to stack the left and right renders for the final render.
I have also updated the documentation on the custom_dataset page to have information on ODS but let me know if there is a better place to put it.

* Updated ODS camera model and render paths

Updates based on suggested code changes including adding a helper function for ODS and improving path creation.
I also added a part where if a user tries to render a video with an output path that doesn't have an extension it will automatically add .mp4 to the end. Otherwise this usually results in an error when there is no file extension when rendering a video.

* Updated formatting for render and cam path scripts

Updated python formatting for render and camera_paths script

* Adding type checking for ODS function and reformatting camera type code

Added the type checking and doc string for the ODS camera rays function and reformatted the cam_type ray if statements.
This commit is contained in:
Cyrus Vachha authored and GitHub committed 2023-05-26 18:01:13 -07:00
1 parent addf477e8c
commit 6993f75dc8
4 files changed
+226 -35

No files matched your search

+7 -1
View File
@@ -139,6 +139,8 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
camera_type = CameraType.FISHEYE
elif camera_path["camera_type"] == "equirectangular":
camera_type = CameraType.EQUIRECTANGULAR
elif camera_path["camera_type"].lower() == "omnidirectional":
camera_type = CameraType.OMNIDIRECTIONALSTEREO_L
else:
camera_type = CameraType.PERSPECTIVE
@@ -149,7 +151,11 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
# pose
c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3]
c2ws.append(c2w)
if camera_type == CameraType.EQUIRECTANGULAR:
if (
camera_type == CameraType.EQUIRECTANGULAR
or camera_type == CameraType.OMNIDIRECTIONALSTEREO_L
or camera_type == CameraType.OMNIDIRECTIONALSTEREO_R
):
fxs.append(image_width / 2)
fys.append(image_height)
else:
+110 -34
View File
@@ -19,7 +19,7 @@ import base64
import math
from dataclasses import dataclass
from enum import Enum, auto
from typing import Dict, List, Optional, Tuple, Union
from typing import Dict, List, Literal, Optional, Tuple, Union
import cv2
import torch
@@ -44,6 +44,8 @@ class CameraType(Enum):
PERSPECTIVE = auto()
FISHEYE = auto()
EQUIRECTANGULAR = auto()
OMNIDIRECTIONALSTEREO_L = auto()
OMNIDIRECTIONALSTEREO_R = auto()
CAMERA_MODEL_TO_TYPE = {
@@ -54,6 +56,8 @@ CAMERA_MODEL_TO_TYPE = {
"OPENCV": CameraType.PERSPECTIVE,
"OPENCV_FISHEYE": CameraType.FISHEYE,
"EQUIRECTANGULAR": CameraType.EQUIRECTANGULAR,
"OMNIDIRECTIONALSTEREO_L": CameraType.OMNIDIRECTIONALSTEREO_L,
"OMNIDIRECTIONALSTEREO_R": CameraType.OMNIDIRECTIONALSTEREO_R,
}
@@ -643,48 +647,120 @@ class Cameras(TensorDataclass):
# directions_stack[2] is the direction for ray in camera coordinates offset by 1 in y
cam_types = torch.unique(self.camera_type, sorted=False)
directions_stack = torch.empty((3,) + num_rays_shape + (3,), device=self.device)
if CameraType.PERSPECTIVE.value in cam_types:
mask = (self.camera_type[true_indices] == CameraType.PERSPECTIVE.value).squeeze(-1) # (num_rays)
c2w = self.camera_to_worlds[true_indices]
assert c2w.shape == num_rays_shape + (3, 4)
def _compute_rays_for_omnidirectional_stereo(
eye: Literal["left", "right"]
) -> Tuple[Float[Tensor, "num_rays_shape 3"], Float[Tensor, "3 num_rays_shape 3"]]:
"""Compute the rays for an omnidirectional stereo camera
Args:
eye: Which eye to compute rays for.
Returns:
A tuple containing the origins and the directions of the rays.
"""
# Directions calculated similarly to equirectangular
ods_cam_type = (
CameraType.OMNIDIRECTIONALSTEREO_R.value if eye == "right" else CameraType.OMNIDIRECTIONALSTEREO_L.value
)
mask = (self.camera_type[true_indices] == ods_cam_type).squeeze(-1)
mask = torch.stack([mask, mask, mask], dim=0)
directions_stack[..., 0][mask] = torch.masked_select(coord_stack[..., 0], mask).float()
directions_stack[..., 1][mask] = torch.masked_select(coord_stack[..., 1], mask).float()
directions_stack[..., 2][mask] = -1.0
if CameraType.FISHEYE.value in cam_types:
mask = (self.camera_type[true_indices] == CameraType.FISHEYE.value).squeeze(-1) # (num_rays)
mask = torch.stack([mask, mask, mask], dim=0)
theta = torch.sqrt(torch.sum(coord_stack**2, dim=-1))
theta = torch.clip(theta, 0.0, math.pi)
sin_theta = torch.sin(theta)
directions_stack[..., 0][mask] = torch.masked_select(coord_stack[..., 0] * sin_theta / theta, mask).float()
directions_stack[..., 1][mask] = torch.masked_select(coord_stack[..., 1] * sin_theta / theta, mask).float()
directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask).float()
if CameraType.EQUIRECTANGULAR.value in cam_types:
mask = (self.camera_type[true_indices] == CameraType.EQUIRECTANGULAR.value).squeeze(-1) # (num_rays)
mask = torch.stack([mask, mask, mask], dim=0)
# For equirect, fx = fy = height = width/2
# Then coord[..., 0] goes from -1 to 1 and coord[..., 1] goes from -1/2 to 1/2
theta = -torch.pi * coord_stack[..., 0] # minus sign for right-handed
theta = -torch.pi * coord_stack[..., 0]
phi = torch.pi * (0.5 - coord_stack[..., 1])
# use spherical in local camera coordinates (+y up, x=0 and z<0 is theta=0)
directions_stack[..., 0][mask] = torch.masked_select(-torch.sin(theta) * torch.sin(phi), mask).float()
directions_stack[..., 1][mask] = torch.masked_select(torch.cos(phi), mask).float()
directions_stack[..., 2][mask] = torch.masked_select(-torch.cos(theta) * torch.sin(phi), mask).float()
for value in cam_types:
if value not in [CameraType.PERSPECTIVE.value, CameraType.FISHEYE.value, CameraType.EQUIRECTANGULAR.value]:
raise ValueError(f"Camera type {value} not supported.")
vr_ipd = 0.064 # IPD in meters (note: scale of NeRF must be true to life and can be adjusted with the Blender add-on)
isRightEye = 1 if eye == "right" else -1
# find ODS camera position
c2w = self.camera_to_worlds[true_indices]
assert c2w.shape == num_rays_shape + (3, 4)
transposedC2W = c2w[0][0].t()
ods_cam_position = transposedC2W[3].repeat(c2w.shape[1], 1)
rotation = c2w[..., :3, :3]
ods_theta = -torch.pi * ((x - cx) / fx)[0]
# local axes of ODS camera
ods_x_axis = torch.tensor([1, 0, 0], device=c2w.device)
ods_z_axis = torch.tensor([0, 0, -1], device=c2w.device)
# circle of ODS ray origins
ods_origins_circle = (
ods_cam_position
+ isRightEye * (vr_ipd / 2.0) * (ods_x_axis.repeat(c2w.shape[1], 1)) * (torch.cos(ods_theta))[:, None]
+ isRightEye * (vr_ipd / 2.0) * (ods_z_axis.repeat(c2w.shape[1], 1)) * (torch.sin(ods_theta))[:, None]
)
# rotate origins to match the camera rotation
for i in range(ods_origins_circle.shape[0]):
ods_origins_circle[i] = rotation[0][0] @ ods_origins_circle[i] + ods_cam_position[0]
ods_origins_circle = ods_origins_circle.unsqueeze(0).repeat(c2w.shape[0], 1, 1)
# assign final camera origins
c2w[..., :3, 3] = ods_origins_circle
return ods_origins_circle, directions_stack
for cam in cam_types:
if CameraType.PERSPECTIVE.value in cam_types:
mask = (self.camera_type[true_indices] == CameraType.PERSPECTIVE.value).squeeze(-1) # (num_rays)
mask = torch.stack([mask, mask, mask], dim=0)
directions_stack[..., 0][mask] = torch.masked_select(coord_stack[..., 0], mask).float()
directions_stack[..., 1][mask] = torch.masked_select(coord_stack[..., 1], mask).float()
directions_stack[..., 2][mask] = -1.0
elif CameraType.FISHEYE.value in cam_types:
mask = (self.camera_type[true_indices] == CameraType.FISHEYE.value).squeeze(-1) # (num_rays)
mask = torch.stack([mask, mask, mask], dim=0)
theta = torch.sqrt(torch.sum(coord_stack**2, dim=-1))
theta = torch.clip(theta, 0.0, math.pi)
sin_theta = torch.sin(theta)
directions_stack[..., 0][mask] = torch.masked_select(
coord_stack[..., 0] * sin_theta / theta, mask
).float()
directions_stack[..., 1][mask] = torch.masked_select(
coord_stack[..., 1] * sin_theta / theta, mask
).float()
directions_stack[..., 2][mask] = -torch.masked_select(torch.cos(theta), mask).float()
elif CameraType.EQUIRECTANGULAR.value in cam_types:
mask = (self.camera_type[true_indices] == CameraType.EQUIRECTANGULAR.value).squeeze(-1) # (num_rays)
mask = torch.stack([mask, mask, mask], dim=0)
# For equirect, fx = fy = height = width/2
# Then coord[..., 0] goes from -1 to 1 and coord[..., 1] goes from -1/2 to 1/2
theta = -torch.pi * coord_stack[..., 0] # minus sign for right-handed
phi = torch.pi * (0.5 - coord_stack[..., 1])
# use spherical in local camera coordinates (+y up, x=0 and z<0 is theta=0)
directions_stack[..., 0][mask] = torch.masked_select(-torch.sin(theta) * torch.sin(phi), mask).float()
directions_stack[..., 1][mask] = torch.masked_select(torch.cos(phi), mask).float()
directions_stack[..., 2][mask] = torch.masked_select(-torch.cos(theta) * torch.sin(phi), mask).float()
elif CameraType.OMNIDIRECTIONALSTEREO_L.value in cam_types:
ods_origins_circle, directions_stack = _compute_rays_for_omnidirectional_stereo("left")
# assign final camera origins
c2w[..., :3, 3] = ods_origins_circle
elif CameraType.OMNIDIRECTIONALSTEREO_R.value in cam_types:
ods_origins_circle, directions_stack = _compute_rays_for_omnidirectional_stereo("right")
# assign final camera origins
c2w[..., :3, 3] = ods_origins_circle
else:
raise ValueError(f"Camera type {cam} not supported.")
assert directions_stack.shape == (3,) + num_rays_shape + (3,)
c2w = self.camera_to_worlds[true_indices]
assert c2w.shape == num_rays_shape + (3, 4)
if camera_opt_to_camera is not None:
c2w = pose_utils.multiply(c2w, camera_opt_to_camera)
rotation = c2w[..., :3, :3] # (..., 3, 3)
+63
View File
@@ -21,6 +21,7 @@ from __future__ import annotations
import json
import os
import struct
import shutil
import sys
from contextlib import ExitStack
from dataclasses import dataclass, field
@@ -59,6 +60,7 @@ from nerfstudio.pipelines.base_pipeline import Pipeline
from nerfstudio.utils import colormaps, install_checks
from nerfstudio.utils.eval_utils import eval_setup
from nerfstudio.utils.rich_utils import CONSOLE, ItersPerSecColumn
from nerfstudio.utils.scripts import run_command
def _render_trajectory_video(
@@ -342,6 +344,22 @@ class RenderCameraPath(BaseRender):
crop_data = get_crop_from_json(camera_path)
camera_path = get_path_from_json(camera_path)
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
# temp folder for writing left and right view renders
temp_folder_path = self.output_path.parent / (self.output_path.stem + "_temp")
Path(temp_folder_path).mkdir(parents=True, exist_ok=True)
left_eye_path = temp_folder_path / "ods_render_Left.mp4"
self.output_path = left_eye_path
CONSOLE.print("[bold green]:goggles: Omni-directional Stereo VR :goggles:")
CONSOLE.print("Rendering left eye view")
# add mp4 suffix to video output if none is specified
if self.output_format == "video" and str(self.output_path.suffix) == "":
self.output_path = self.output_path.with_suffix(".mp4")
_render_trajectory_video(
pipeline,
camera_path,
@@ -356,6 +374,51 @@ class RenderCameraPath(BaseRender):
colormap_options=self.colormap_options,
)
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
# declare paths for left and right renders
left_eye_path = self.output_path
right_eye_path = left_eye_path.parent / "ods_render_Right.mp4"
self.output_path = right_eye_path
camera_path.camera_type[0] = CameraType.OMNIDIRECTIONALSTEREO_R.value
CONSOLE.print("Rendering right eye view")
_render_trajectory_video(
pipeline,
camera_path,
output_filename=self.output_path,
rendered_output_names=self.rendered_output_names,
rendered_resolution_scaling_factor=1.0 / self.downscale_factor,
crop_data=crop_data,
seconds=seconds,
output_format=self.output_format,
image_format=self.image_format,
jpeg_quality=self.jpeg_quality,
colormap_options=self.colormap_options,
)
# stack the left and right eye renders for final output
self.output_path = Path(str(left_eye_path.parent)[:-5] + ".mp4")
ffmpeg_ods_command = ""
if self.output_format == "video":
ffmpeg_ods_command = f'ffmpeg -y -i "{left_eye_path}" -i "{right_eye_path}" -filter_complex "[0:v]pad=iw:2*ih[int];[int][1:v]overlay=0:h" -c:v libx264 -crf 23 -preset veryfast "{self.output_path}"'
run_command(ffmpeg_ods_command, verbose=False)
if self.output_format == "images":
# create a folder for the stacked renders
self.output_path = Path(str(left_eye_path.parent)[:-5])
self.output_path.mkdir(parents=True, exist_ok=True)
if self.image_format == "png":
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.png")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.png")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.png"}"'
elif self.image_format == "jpeg":
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.jpg")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.jpg")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.jpg"}"'
run_command(ffmpeg_ods_command, verbose=False)
# remove the temp files directory
if str(left_eye_path.parent)[-5:] == "_temp":
shutil.rmtree(left_eye_path.parent, ignore_errors=True)
CONSOLE.print("[bold green]Final ODS Render Complete")
@dataclass
class RenderInterpolated(BaseRender):