mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 14:45:38 +08:00
Render VR180 video (#2261)
* Render VR180 video Adding the VR180 camera model to render VR180 video and images. I condensed the omnidirectional stereo and VR180 render post processing in the render script. I have updated the documentation for VR180 support. * Condensing formatting of camera types Condensing the if statement for camera type intrinsics.
This commit is contained in:
1 parent
4f2157cd58
commit
8682e481b1
5 files changed
+187
-47
No files matched your search
@@ -141,6 +141,8 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
|
||||
camera_type = CameraType.EQUIRECTANGULAR
|
||||
elif camera_path["camera_type"].lower() == "omnidirectional":
|
||||
camera_type = CameraType.OMNIDIRECTIONALSTEREO_L
|
||||
elif camera_path["camera_type"].lower() == "vr180":
|
||||
camera_type = CameraType.VR180_L
|
||||
else:
|
||||
camera_type = CameraType.PERSPECTIVE
|
||||
|
||||
@@ -151,11 +153,13 @@ def get_path_from_json(camera_path: Dict[str, Any]) -> Cameras:
|
||||
# pose
|
||||
c2w = torch.tensor(camera["camera_to_world"]).view(4, 4)[:3]
|
||||
c2ws.append(c2w)
|
||||
if (
|
||||
camera_type == CameraType.EQUIRECTANGULAR
|
||||
or camera_type == CameraType.OMNIDIRECTIONALSTEREO_L
|
||||
or camera_type == CameraType.OMNIDIRECTIONALSTEREO_R
|
||||
):
|
||||
if camera_type in [
|
||||
CameraType.EQUIRECTANGULAR,
|
||||
CameraType.OMNIDIRECTIONALSTEREO_L,
|
||||
CameraType.OMNIDIRECTIONALSTEREO_R,
|
||||
CameraType.VR180_L,
|
||||
CameraType.VR180_R,
|
||||
]:
|
||||
fxs.append(image_width / 2)
|
||||
fys.append(image_height)
|
||||
else:
|
||||
|
||||
@@ -46,6 +46,8 @@ class CameraType(Enum):
|
||||
EQUIRECTANGULAR = auto()
|
||||
OMNIDIRECTIONALSTEREO_L = auto()
|
||||
OMNIDIRECTIONALSTEREO_R = auto()
|
||||
VR180_L = auto()
|
||||
VR180_R = auto()
|
||||
|
||||
|
||||
CAMERA_MODEL_TO_TYPE = {
|
||||
@@ -58,6 +60,8 @@ CAMERA_MODEL_TO_TYPE = {
|
||||
"EQUIRECTANGULAR": CameraType.EQUIRECTANGULAR,
|
||||
"OMNIDIRECTIONALSTEREO_L": CameraType.OMNIDIRECTIONALSTEREO_L,
|
||||
"OMNIDIRECTIONALSTEREO_R": CameraType.OMNIDIRECTIONALSTEREO_R,
|
||||
"VR180_L": CameraType.VR180_L,
|
||||
"VR180_R": CameraType.VR180_R,
|
||||
}
|
||||
|
||||
|
||||
@@ -708,6 +712,60 @@ class Cameras(TensorDataclass):
|
||||
|
||||
return ods_origins_circle, directions_stack
|
||||
|
||||
def _compute_rays_for_vr180(
|
||||
eye: Literal["left", "right"]
|
||||
) -> Tuple[Float[Tensor, "num_rays_shape 3"], Float[Tensor, "3 num_rays_shape 3"]]:
|
||||
"""Compute the rays for a VR180 camera
|
||||
|
||||
Args:
|
||||
eye: Which eye to compute rays for.
|
||||
|
||||
Returns:
|
||||
A tuple containing the origins and the directions of the rays.
|
||||
"""
|
||||
# Directions calculated similarly to equirectangular
|
||||
vr180_cam_type = CameraType.VR180_R.value if eye == "right" else CameraType.VR180_L.value
|
||||
mask = (self.camera_type[true_indices] == vr180_cam_type).squeeze(-1)
|
||||
mask = torch.stack([mask, mask, mask], dim=0)
|
||||
|
||||
# adjusting theta range to +/-90 deg
|
||||
theta = -torch.pi * ((x - cx) / (fx * 2))[0]
|
||||
phi = torch.pi * (0.5 - coord_stack[..., 1])
|
||||
|
||||
directions_stack[..., 0][mask] = torch.masked_select(-torch.sin(theta) * torch.sin(phi), mask).float()
|
||||
directions_stack[..., 1][mask] = torch.masked_select(torch.cos(phi), mask).float()
|
||||
directions_stack[..., 2][mask] = torch.masked_select(-torch.cos(theta) * torch.sin(phi), mask).float()
|
||||
|
||||
vr_ipd = 0.064 # IPD in meters (note: scale of NeRF must be true to life and can be adjusted with the Blender add-on)
|
||||
isRightEye = 1 if eye == "right" else -1
|
||||
|
||||
# find VR180 camera position
|
||||
c2w = self.camera_to_worlds[true_indices]
|
||||
assert c2w.shape == num_rays_shape + (3, 4)
|
||||
transposedC2W = c2w[0][0].t()
|
||||
vr180_cam_position = transposedC2W[3].repeat(c2w.shape[1], 1)
|
||||
|
||||
rotation = c2w[..., :3, :3]
|
||||
|
||||
-torch.pi * ((x - cx) / fx)[0]
|
||||
|
||||
# interocular axis of the VR180 camera
|
||||
vr180_x_axis = torch.tensor([1, 0, 0], device=c2w.device)
|
||||
|
||||
# VR180 ray origins of horizontal offset
|
||||
vr180_origins = isRightEye * (vr_ipd / 2.0) * (vr180_x_axis.repeat(c2w.shape[1], 1))
|
||||
|
||||
# rotate origins to match the camera rotation
|
||||
for i in range(vr180_origins.shape[0]):
|
||||
vr180_origins[i] = rotation[0][0] @ vr180_origins[i] + vr180_cam_position[0]
|
||||
|
||||
vr180_origins = vr180_origins.unsqueeze(0).repeat(c2w.shape[0], 1, 1)
|
||||
|
||||
# assign final camera origins
|
||||
c2w[..., :3, 3] = vr180_origins
|
||||
|
||||
return vr180_origins, directions_stack
|
||||
|
||||
for cam in cam_types:
|
||||
if CameraType.PERSPECTIVE.value in cam_types:
|
||||
mask = (self.camera_type[true_indices] == CameraType.PERSPECTIVE.value).squeeze(-1) # (num_rays)
|
||||
@@ -755,6 +813,17 @@ class Cameras(TensorDataclass):
|
||||
ods_origins_circle, directions_stack = _compute_rays_for_omnidirectional_stereo("right")
|
||||
# assign final camera origins
|
||||
c2w[..., :3, 3] = ods_origins_circle
|
||||
|
||||
elif CameraType.VR180_L.value in cam_types:
|
||||
vr180_origins, directions_stack = _compute_rays_for_vr180("left")
|
||||
# assign final camera origins
|
||||
c2w[..., :3, 3] = vr180_origins
|
||||
|
||||
elif CameraType.VR180_R.value in cam_types:
|
||||
vr180_origins, directions_stack = _compute_rays_for_vr180("right")
|
||||
# assign final camera origins
|
||||
c2w[..., :3, 3] = vr180_origins
|
||||
|
||||
else:
|
||||
raise ValueError(f"Camera type {cam} not supported.")
|
||||
|
||||
|
||||
@@ -344,16 +344,23 @@ class RenderCameraPath(BaseRender):
|
||||
crop_data = get_crop_from_json(camera_path)
|
||||
camera_path = get_path_from_json(camera_path)
|
||||
|
||||
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
|
||||
if (
|
||||
camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value
|
||||
or camera_path.camera_type[0] == CameraType.VR180_L.value
|
||||
):
|
||||
# temp folder for writing left and right view renders
|
||||
temp_folder_path = self.output_path.parent / (self.output_path.stem + "_temp")
|
||||
|
||||
Path(temp_folder_path).mkdir(parents=True, exist_ok=True)
|
||||
left_eye_path = temp_folder_path / "ods_render_Left.mp4"
|
||||
left_eye_path = temp_folder_path / "render_left.mp4"
|
||||
|
||||
self.output_path = left_eye_path
|
||||
|
||||
CONSOLE.print("[bold green]:goggles: Omni-directional Stereo VR :goggles:")
|
||||
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
|
||||
CONSOLE.print("[bold green]:goggles: Omni-directional Stereo VR :goggles:")
|
||||
else:
|
||||
CONSOLE.print("[bold green]:goggles: VR180 :goggles:")
|
||||
|
||||
CONSOLE.print("Rendering left eye view")
|
||||
|
||||
# add mp4 suffix to video output if none is specified
|
||||
@@ -374,14 +381,21 @@ class RenderCameraPath(BaseRender):
|
||||
colormap_options=self.colormap_options,
|
||||
)
|
||||
|
||||
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
|
||||
if (
|
||||
camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value
|
||||
or camera_path.camera_type[0] == CameraType.VR180_L.value
|
||||
):
|
||||
# declare paths for left and right renders
|
||||
|
||||
left_eye_path = self.output_path
|
||||
right_eye_path = left_eye_path.parent / "ods_render_Right.mp4"
|
||||
right_eye_path = left_eye_path.parent / "render_right.mp4"
|
||||
|
||||
self.output_path = right_eye_path
|
||||
camera_path.camera_type[0] = CameraType.OMNIDIRECTIONALSTEREO_R.value
|
||||
|
||||
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_L.value:
|
||||
camera_path.camera_type[0] = CameraType.OMNIDIRECTIONALSTEREO_R.value
|
||||
else:
|
||||
camera_path.camera_type[0] = CameraType.VR180_R.value
|
||||
|
||||
CONSOLE.print("Rendering right eye view")
|
||||
_render_trajectory_video(
|
||||
@@ -398,26 +412,49 @@ class RenderCameraPath(BaseRender):
|
||||
colormap_options=self.colormap_options,
|
||||
)
|
||||
|
||||
# stack the left and right eye renders for final output
|
||||
self.output_path = Path(str(left_eye_path.parent)[:-5] + ".mp4")
|
||||
ffmpeg_ods_command = ""
|
||||
if self.output_format == "video":
|
||||
ffmpeg_ods_command = f'ffmpeg -y -i "{left_eye_path}" -i "{right_eye_path}" -filter_complex "[0:v]pad=iw:2*ih[int];[int][1:v]overlay=0:h" -c:v libx264 -crf 23 -preset veryfast "{self.output_path}"'
|
||||
run_command(ffmpeg_ods_command, verbose=False)
|
||||
if self.output_format == "images":
|
||||
# create a folder for the stacked renders
|
||||
self.output_path = Path(str(left_eye_path.parent)[:-5])
|
||||
self.output_path.mkdir(parents=True, exist_ok=True)
|
||||
if self.image_format == "png":
|
||||
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.png")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.png")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.png"}"'
|
||||
elif self.image_format == "jpeg":
|
||||
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.jpg")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.jpg")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.jpg"}"'
|
||||
run_command(ffmpeg_ods_command, verbose=False)
|
||||
|
||||
# remove the temp files directory
|
||||
if str(left_eye_path.parent)[-5:] == "_temp":
|
||||
shutil.rmtree(left_eye_path.parent, ignore_errors=True)
|
||||
CONSOLE.print("[bold green]Final ODS Render Complete")
|
||||
if camera_path.camera_type[0] == CameraType.OMNIDIRECTIONALSTEREO_R.value:
|
||||
# stack the left and right eye renders vertically for ODS final output
|
||||
ffmpeg_ods_command = ""
|
||||
if self.output_format == "video":
|
||||
ffmpeg_ods_command = f'ffmpeg -y -i "{left_eye_path}" -i "{right_eye_path}" -filter_complex "[0:v]pad=iw:2*ih[int];[int][1:v]overlay=0:h" -c:v libx264 -crf 23 -preset veryfast "{self.output_path}"'
|
||||
run_command(ffmpeg_ods_command, verbose=False)
|
||||
if self.output_format == "images":
|
||||
# create a folder for the stacked renders
|
||||
self.output_path = Path(str(left_eye_path.parent)[:-5])
|
||||
self.output_path.mkdir(parents=True, exist_ok=True)
|
||||
if self.image_format == "png":
|
||||
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.png")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.png")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.png"}"'
|
||||
elif self.image_format == "jpeg":
|
||||
ffmpeg_ods_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.jpg")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.jpg")}" -filter_complex vstack -start_number 0 "{str(self.output_path)+"//%05d.jpg"}"'
|
||||
run_command(ffmpeg_ods_command, verbose=False)
|
||||
|
||||
# remove the temp files directory
|
||||
if str(left_eye_path.parent)[-5:] == "_temp":
|
||||
shutil.rmtree(left_eye_path.parent, ignore_errors=True)
|
||||
CONSOLE.print("[bold green]Final ODS Render Complete")
|
||||
else:
|
||||
# stack the left and right eye renders horizontally for VR180 final output
|
||||
self.output_path = Path(str(left_eye_path.parent)[:-5] + ".mp4")
|
||||
ffmpeg_vr180_command = ""
|
||||
if self.output_format == "video":
|
||||
ffmpeg_vr180_command = f'ffmpeg -y -i "{right_eye_path}" -i "{left_eye_path}" -filter_complex "[1:v]hstack=inputs=2" -c:a copy "{self.output_path}"'
|
||||
run_command(ffmpeg_vr180_command, verbose=False)
|
||||
if self.output_format == "images":
|
||||
# create a folder for the stacked renders
|
||||
self.output_path = Path(str(left_eye_path.parent)[:-5])
|
||||
self.output_path.mkdir(parents=True, exist_ok=True)
|
||||
if self.image_format == "png":
|
||||
ffmpeg_vr180_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.png")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.png")}" -filter_complex hstack -start_number 0 "{str(self.output_path)+"//%05d.png"}"'
|
||||
elif self.image_format == "jpeg":
|
||||
ffmpeg_vr180_command = f'ffmpeg -y -pattern_type glob -i "{str(left_eye_path.with_suffix("") / "*.jpg")}" -pattern_type glob -i "{str(right_eye_path.with_suffix("") / "*.jpg")}" -filter_complex hstack -start_number 0 "{str(self.output_path)+"//%05d.jpg"}"'
|
||||
run_command(ffmpeg_vr180_command, verbose=False)
|
||||
|
||||
# remove the temp files directory
|
||||
if str(left_eye_path.parent)[-5:] == "_temp":
|
||||
shutil.rmtree(left_eye_path.parent, ignore_errors=True)
|
||||
CONSOLE.print("[bold green]Final VR180 Render Complete")
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
Reference in new issue
Block a user