Render VR180 video (#2261)

* Render VR180 video

Adding the VR180 camera model to render VR180 video and images. I condensed the omnidirectional stereo and VR180 render post processing in the render script. I have updated the documentation for VR180 support.

* Condensing formatting of camera types

Condensing the if statement for camera type intrinsics.
This commit is contained in:
Cyrus Vachha authored and GitHub committed 2023-07-24 07:26:20 -07:00
1 parent 4f2157cd58
commit 8682e481b1
5 files changed
+187 -47

No files matched your search

+9 -5
View File
@@ -141,6 +141,8 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
camera_type = CameraType.EQUIRECTANGULAR
elif camera_path["camera_type"].lower() == "omnidirectional":
camera_type = CameraType.OMNIDIRECTIONALSTEREO_L
elif camera_path["camera_type"].lower() == "vr180":
camera_type = CameraType.VR180_L
else:
camera_type = CameraType.PERSPECTIVE
@@ -151,11 +153,13 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
# pose
c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3]
c2ws.append(c2w)
if (
camera_type == CameraType.EQUIRECTANGULAR
or camera_type == CameraType.OMNIDIRECTIONALSTEREO_L
or camera_type == CameraType.OMNIDIRECTIONALSTEREO_R
):
if camera_type in [
CameraType.EQUIRECTANGULAR,
CameraType.OMNIDIRECTIONALSTEREO_L,
CameraType.OMNIDIRECTIONALSTEREO_R,
CameraType.VR180_L,
CameraType.VR180_R,
]:
fxs.append(image_width / 2)
fys.append(image_height)
else:
+69
View File
@@ -46,6 +46,8 @@ class CameraType(Enum):
EQUIRECTANGULAR = auto()
OMNIDIRECTIONALSTEREO_L = auto()
OMNIDIRECTIONALSTEREO_R = auto()
VR180_L = auto()
VR180_R = auto()
CAMERA_MODEL_TO_TYPE = {
@@ -58,6 +60,8 @@ CAMERA_MODEL_TO_TYPE = {
"EQUIRECTANGULAR": CameraType.EQUIRECTANGULAR,
"OMNIDIRECTIONALSTEREO_L": CameraType.OMNIDIRECTIONALSTEREO_L,
"OMNIDIRECTIONALSTEREO_R": CameraType.OMNIDIRECTIONALSTEREO_R,
"VR180_L": CameraType.VR180_L,
"VR180_R": CameraType.VR180_R,
}
@@ -708,6 +712,60 @@ class Cameras(TensorDataclass):
return ods_origins_circle, directions_stack
def _compute_rays_for_vr180(
eye: Literal["left", "right"]
) -> Tuple[Float[Tensor, "num_rays_shape 3"], Float[Tensor, "3 num_rays_shape 3"]]:
"""Compute the rays for a VR180 camera
Args:
eye: Which eye to compute rays for.
Returns:
A tuple containing the origins and the directions of the rays.
"""
# Directions calculated similarly to equirectangular
vr180_cam_type = CameraType.VR180_R.value if eye == "right" else CameraType.VR180_L.value
mask = (self.camera_type[true_indices] == vr180_cam_type).squeeze(-1)
mask = torch.stack([mask, mask, mask], dim=0)
# adjusting theta range to +/-90 deg
theta = -torch.pi * ((x - cx) / (fx * 2))[0]
phi = torch.pi * (0.5 - coord_stack[..., 1])
directions_stack[..., 0][mask] = torch.masked_select(-torch.sin(theta) * torch.sin(phi), mask).float()
directions_stack[..., 1][mask] = torch.masked_select(torch.cos(phi), mask).float()
directions_stack[..., 2][mask] = torch.masked_select(-torch.cos(theta) * torch.sin(phi), mask).float()
vr_ipd = 0.064 # IPD in meters (note: scale of NeRF must be true to life and can be adjusted with the Blender add-on)
isRightEye = 1 if eye == "right" else -1
# find VR180 camera position
c2w = self.camera_to_worlds[true_indices]
assert c2w.shape == num_rays_shape + (3, 4)
transposedC2W = c2w[0][0].t()
vr180_cam_position = transposedC2W[3].repeat(c2w.shape[1], 1)
rotation = c2w[..., :3, :3]
-torch.pi * ((x - cx) / fx)[0]
# interocular axis of the VR180 camera
vr180_x_axis = torch.tensor([1, 0, 0], device=c2w.device)
# VR180 ray origins of horizontal offset
vr180_origins = isRightEye * (vr_ipd / 2.0) * (vr180_x_axis.repeat(c2w.shape[1], 1))
# rotate origins to match the camera rotation
for i in range(vr180_origins.shape[0]):
vr180_origins[i] = rotation[0][0] @ vr180_origins[i] + vr180_cam_position[0]
vr180_origins = vr180_origins.unsqueeze(0).repeat(c2w.shape[0], 1, 1)
# assign final camera origins
c2w[..., :3, 3] = vr180_origins
return vr180_origins, directions_stack
for cam in cam_types:
if CameraType.PERSPECTIVE.value in cam_types:
mask = (self.camera_type[true_indices] == CameraType.PERSPECTIVE.value).squeeze(-1) # (num_rays)
@@ -755,6 +813,17 @@ class Cameras(TensorDataclass):
ods_origins_circle, directions_stack = _compute_rays_for_omnidirectional_stereo("right")
# assign final camera origins
c2w[..., :3, 3] = ods_origins_circle
elif CameraType.VR180_L.value in cam_types:
vr180_origins, directions_stack = _compute_rays_for_vr180("left")
# assign final camera origins
c2w[..., :3, 3] = vr180_origins
elif CameraType.VR180_R.value in cam_types:
vr180_origins, directions_stack = _compute_rays_for_vr180("right")
# assign final camera origins
c2w[..., :3, 3] = vr180_origins
else:
raise ValueError(f"Camera type {cam} not supported.")
+61 -24
View File
@@ -344,16 +344,23 @@ class RenderCameraPath(BaseRender):
crop_data = get_crop_from_json(camera_path)
camera_path = get_path_from_json(camera_path)
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
if (
camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value
or camera_path.camera_type[0] == CameraType.VR180_L.value
):
# temp folder for writing left and right view renders
temp_folder_path = self.output_path.parent / (self.output_path.stem + "_temp")
Path(temp_folder_path).mkdir(parents=True, exist_ok=True)
left_eye_path = temp_folder_path / "ods_render_Left.mp4"
left_eye_path = temp_folder_path / "render_left.mp4"
self.output_path = left_eye_path
CONSOLE.print("[bold green]:goggles: Omni-directional Stereo VR :goggles:")
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
CONSOLE.print("[bold green]:goggles: Omni-directional Stereo VR :goggles:")
else:
CONSOLE.print("[bold green]:goggles: VR180 :goggles:")
CONSOLE.print("Rendering left eye view")
# add mp4 suffix to video output if none is specified
@@ -374,14 +381,21 @@ class RenderCameraPath(BaseRender):
colormap_options=self.colormap_options,
)
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
if (
camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value
or camera_path.camera_type[0] == CameraType.VR180_L.value
):
# declare paths for left and right renders
left_eye_path = self.output_path
right_eye_path = left_eye_path.parent / "ods_render_Right.mp4"
right_eye_path = left_eye_path.parent / "render_right.mp4"
self.output_path = right_eye_path
camera_path.camera_type[0] = CameraType.OMNIDIRECTIONALSTEREO_R.value
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
camera_path.camera_type[0] = CameraType.OMNIDIRECTIONALSTEREO_R.value
else:
camera_path.camera_type[0] = CameraType.VR180_R.value
CONSOLE.print("Rendering right eye view")
_render_trajectory_video(
@@ -398,26 +412,49 @@ class RenderCameraPath(BaseRender):
colormap_options=self.colormap_options,
)
# stack the left and right eye renders for final output
self.output_path = Path(str(left_eye_path.parent)[:-5] + ".mp4")
ffmpeg_ods_command = ""
if self.output_format == "video":
ffmpeg_ods_command = f'ffmpeg -y -i "{left_eye_path}" -i "{right_eye_path}" -filter_complex "[0:v]pad=iw:2*ih[int];[int][1:v]overlay=0:h" -c:v libx264 -crf 23 -preset veryfast "{self.output_path}"'
run_command(ffmpeg_ods_command, verbose=False)
if self.output_format == "images":
# create a folder for the stacked renders
self.output_path = Path(str(left_eye_path.parent)[:-5])
self.output_path.mkdir(parents=True, exist_ok=True)
if self.image_format == "png":
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.png")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.png")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.png"}"'
elif self.image_format == "jpeg":
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.jpg")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.jpg")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.jpg"}"'
run_command(ffmpeg_ods_command, verbose=False)
# remove the temp files directory
if str(left_eye_path.parent)[-5:] == "_temp":
shutil.rmtree(left_eye_path.parent, ignore_errors=True)
CONSOLE.print("[bold green]Final ODS Render Complete")
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_R.value:
# stack the left and right eye renders vertically for ODS final output
ffmpeg_ods_command = ""
if self.output_format == "video":
ffmpeg_ods_command = f'ffmpeg -y -i "{left_eye_path}" -i "{right_eye_path}" -filter_complex "[0:v]pad=iw:2*ih[int];[int][1:v]overlay=0:h" -c:v libx264 -crf 23 -preset veryfast "{self.output_path}"'
run_command(ffmpeg_ods_command, verbose=False)
if self.output_format == "images":
# create a folder for the stacked renders
self.output_path = Path(str(left_eye_path.parent)[:-5])
self.output_path.mkdir(parents=True, exist_ok=True)
if self.image_format == "png":
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.png")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.png")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.png"}"'
elif self.image_format == "jpeg":
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.jpg")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.jpg")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.jpg"}"'
run_command(ffmpeg_ods_command, verbose=False)
# remove the temp files directory
if str(left_eye_path.parent)[-5:] == "_temp":
shutil.rmtree(left_eye_path.parent, ignore_errors=True)
CONSOLE.print("[bold green]Final ODS Render Complete")
else:
# stack the left and right eye renders horizontally for VR180 final output
self.output_path = Path(str(left_eye_path.parent)[:-5] + ".mp4")
ffmpeg_vr180_command = ""
if self.output_format == "video":
ffmpeg_vr180_command = f'ffmpeg -y -i "{right_eye_path}" -i "{left_eye_path}" -filter_complex "[1:v]hstack=inputs=2" -c:a copy "{self.output_path}"'
run_command(ffmpeg_vr180_command, verbose=False)
if self.output_format == "images":
# create a folder for the stacked renders
self.output_path = Path(str(left_eye_path.parent)[:-5])
self.output_path.mkdir(parents=True, exist_ok=True)
if self.image_format == "png":
ffmpeg_vr180_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.png")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.png")}" -filter_complex hstack -start_number 0 "{str(self.output_path)+"//%05d.png"}"'
elif self.image_format == "jpeg":
ffmpeg_vr180_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.jpg")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.jpg")}" -filter_complex hstack -start_number 0 "{str(self.output_path)+"//%05d.jpg"}"'
run_command(ffmpeg_vr180_command, verbose=False)
# remove the temp files directory
if str(left_eye_path.parent)[-5:] == "_temp":
shutil.rmtree(left_eye_path.parent, ignore_errors=True)
CONSOLE.print("[bold green]Final VR180 Render Complete")
@dataclass