From 319a2b965c4a578a413fb81ad6f90e6ac4fae4c5 Mon Sep 17 00:00:00 2001 From: machenmusik Date: Fri, 3 Feb 2023 16:38:40 -0500 Subject: [PATCH] New config parameter --viewer.codec defaults to VP8, but allows H264 (#1343) * New config parameter --viewer.codec defaults to H264 rather than VP8 * make quantize clearer * change codec to Literal default is VP8 due to libx264 log spam, see https://github.com/nerfstudio-project/nerfstudio/pull/1343#issuecomment-1416235500 * isort * pylint, including python 3.7 compatibility --- nerfstudio/configs/base_config.py | 4 ++++ nerfstudio/viewer/server/utils.py | 5 +++-- nerfstudio/viewer/server/viewer_utils.py | 10 ++++++++-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index 548324069..a85b5ddaa 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -23,6 +23,8 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, List, Optional, Tuple, Type +from typing_extensions import Literal + # model instances from nerfstudio.utils import writer @@ -149,3 +151,5 @@ class ViewerConfig(PrintableConfig): skip_openrelay: bool = False """Avoid using openrelay to communicate with the viewer. Try disabling if you have trouble connecting to the viewer""" + codec: Literal["H264", "VP8"] = "VP8" + """Video codec that viewer will use.""" diff --git a/nerfstudio/viewer/server/utils.py b/nerfstudio/viewer/server/utils.py index b0763b6d1..a21b02d82 100644 --- a/nerfstudio/viewer/server/utils.py +++ b/nerfstudio/viewer/server/utils.py @@ -63,7 +63,7 @@ def three_js_perspective_camera_focal_length(fov: float, image_height: int): def get_intrinsics_matrix_and_camera_to_world_h( - camera_object: Dict[str, Any], image_height: int + camera_object: Dict[str, Any], image_height: int, image_width: Optional[int] = None ) -> Tuple[torch.Tensor, torch.Tensor]: """Returns the camera intrinsics matrix and the camera to world homogeneous matrix. @@ -74,7 +74,8 @@ def get_intrinsics_matrix_and_camera_to_world_h( # intrinsics fov = camera_object["fov"] aspect = camera_object["aspect"] - image_width = aspect * image_height + if image_width is None: + image_width = aspect * image_height pp_w = image_width / 2.0 pp_h = image_height / 2.0 if (camera_object["camera_type"] == "perspective") | (camera_object["camera_type"] == "fisheye"): diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index 93dc908b6..b59381f26 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -689,7 +689,8 @@ class ViewerState: video = SingleFrameStreamTrack() self.video_tracks.add(video) video_sender = pc.addTrack(video) - force_codec(pc, video_sender, "video/VP8") + print(f"viewer using video/{self.config.codec}") + force_codec(pc, video_sender, f"video/{self.config.codec}") await pc.setRemoteDescription(offer) answer = await pc.createAnswer() @@ -831,6 +832,11 @@ class ViewerState: if image_width > self.max_resolution: image_width = self.max_resolution image_height = int(image_width / aspect_ratio) + if self.config.codec != "VP8": + # force even values to allow hardware encoder usage + quantize = 2 + image_width = int(image_width / quantize) * quantize + image_height = int(image_height / quantize) * quantize return image_height, image_width def _process_invalid_output(self, output_type: str) -> str: @@ -904,7 +910,7 @@ class ViewerState: return intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h( - camera_object, image_height=image_height + camera_object, image_height=image_height, image_width=image_width ) camera_to_world = camera_to_world_h[:3, :]