mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 14:23:57 +08:00
571 lines
18 KiB
Python
571 lines
18 KiB
Python
# Copyright 2022 The Nerfstudio Team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
""" This module contains the MessageApi class, which is the interface for sending messages to the Viewer"""
|
|
# pylint: disable=protected-access
|
|
# pylint: disable=too-many-public-methods
|
|
|
|
from __future__ import annotations
|
|
|
|
import abc
|
|
import base64
|
|
import contextlib
|
|
import io
|
|
import time
|
|
from pathlib import Path
|
|
from typing import (
|
|
TYPE_CHECKING,
|
|
Any,
|
|
Callable,
|
|
Dict,
|
|
Generator,
|
|
List,
|
|
Optional,
|
|
Tuple,
|
|
TypeVar,
|
|
cast,
|
|
overload,
|
|
)
|
|
|
|
import imageio.v3 as iio
|
|
import numpy as onp
|
|
import numpy.typing as onpt
|
|
from typing_extensions import Literal, LiteralString, ParamSpec, assert_never
|
|
|
|
from nerfstudio.data.scene_box import SceneBox
|
|
|
|
from . import messages
|
|
from .gui import GuiHandle, GuiSelectHandle, _GuiHandleState
|
|
|
|
if TYPE_CHECKING:
|
|
from viser.infra import ClientId
|
|
|
|
|
|
P = ParamSpec("P")
|
|
|
|
|
|
def _colors_to_uint8(colors: onp.ndarray) -> onpt.NDArray[onp.uint8]:
|
|
"""Convert intensity values to uint8. We assume the range [0,1] for floats, and
|
|
[0,255] for integers."""
|
|
if colors.dtype != onp.uint8:
|
|
if onp.issubdtype(colors.dtype, onp.floating):
|
|
colors = onp.clip(colors * 255.0, 0, 255).astype(onp.uint8)
|
|
if onp.issubdtype(colors.dtype, onp.integer):
|
|
colors = onp.clip(colors, 0, 255).astype(onp.uint8)
|
|
return colors
|
|
|
|
|
|
def _encode_image_base64(
|
|
image: onp.ndarray,
|
|
file_format: Literal["png", "jpeg"],
|
|
quality: Optional[int] = None,
|
|
) -> Tuple[Literal["image/png", "image/jpeg"], str]:
|
|
"""Encode an image as a base64 string.
|
|
|
|
Args:
|
|
image: The image to encode.
|
|
file_format: The format to encode the image as.
|
|
quality: The quality to encode the image as. Only used for JPEG.
|
|
|
|
Returns:
|
|
A tuple of the media type and the base64-encoded image.
|
|
"""
|
|
|
|
media_type: Literal["image/png", "image/jpeg"]
|
|
image = _colors_to_uint8(image)
|
|
with io.BytesIO() as data_buffer:
|
|
if file_format == "png":
|
|
media_type = "image/png"
|
|
iio.imwrite(data_buffer, image, extension=".png")
|
|
elif file_format == "jpeg":
|
|
media_type = "image/jpeg"
|
|
iio.imwrite(
|
|
data_buffer,
|
|
image[..., :3], # Strip alpha.
|
|
extension=".jpeg",
|
|
quality=75 if quality is None else quality,
|
|
)
|
|
else:
|
|
assert_never(file_format)
|
|
|
|
base64_data = base64.b64encode(data_buffer.getvalue()).decode("ascii")
|
|
|
|
return media_type, base64_data
|
|
|
|
|
|
TVector = TypeVar("TVector", bound=tuple)
|
|
|
|
|
|
def _cast_vector(vector: TVector | onp.ndarray, length: int) -> TVector:
|
|
if isinstance(vector, tuple):
|
|
assert len(vector) == length
|
|
return cast(TVector, vector)
|
|
assert cast(onp.ndarray, vector).shape == (length,)
|
|
return cast(TVector, tuple(map(float, vector)))
|
|
|
|
|
|
T = TypeVar("T")
|
|
IntOrFloat = TypeVar("IntOrFloat", int, float)
|
|
TLiteralString = TypeVar("TLiteralString", bound=LiteralString)
|
|
|
|
|
|
class MessageApi(abc.ABC):
|
|
"""Interface for all commands we can use to send messages over a websocket connection.
|
|
|
|
Should be implemented by both our global server object (for broadcasting) and by
|
|
invidividual clients."""
|
|
|
|
def __init__(self) -> None:
|
|
self._handle_state_from_gui_name: Dict[str, _GuiHandleState[Any]] = {}
|
|
self._gui_folder_labels: List[str] = []
|
|
|
|
@abc.abstractmethod
|
|
def _queue(self, message: messages.NerfstudioMessage) -> None:
|
|
"""Abstract method for sending messages."""
|
|
...
|
|
|
|
@contextlib.contextmanager
|
|
def gui_folder(self, label: str) -> Generator[None, None, None]:
|
|
"""Context for placing all GUI elements into a particular folder.
|
|
|
|
We currently only support one folder level.
|
|
|
|
Args:
|
|
label: The label for the folder.
|
|
"""
|
|
self._gui_folder_labels.append(label)
|
|
yield
|
|
assert self._gui_folder_labels.pop() == label
|
|
|
|
def add_gui_button(self, name: str) -> GuiHandle[bool]:
|
|
"""Add a button to the GUI. The value of this input is set to `True` every time
|
|
it is clicked; to detect clicks, we can manually set it back to `False`.
|
|
|
|
Currently, all button names need to be unique.
|
|
|
|
Args:
|
|
name: The name of the gui element.
|
|
"""
|
|
return self._add_gui_impl(
|
|
name,
|
|
initial_value=False,
|
|
leva_conf={"type": "BUTTON", "settings": {}},
|
|
is_button=True,
|
|
)
|
|
|
|
def add_gui_checkbox(self, name: str, initial_value: bool) -> GuiHandle[bool]:
|
|
"""Add a checkbox to the GUI.
|
|
|
|
Args:
|
|
name: The name of the checkbox.
|
|
initial_value: The initial value of the checkbox.
|
|
"""
|
|
assert isinstance(initial_value, bool)
|
|
return self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
initial_value,
|
|
leva_conf={"value": initial_value, "label": name},
|
|
)
|
|
|
|
def add_gui_text(self, name: str, initial_value: str) -> GuiHandle[str]:
|
|
"""Add a text input to the GUI.
|
|
|
|
Args:
|
|
name: The name of the text input.
|
|
initial_value: The initial value of the text input.
|
|
"""
|
|
assert isinstance(initial_value, str)
|
|
return self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
initial_value,
|
|
leva_conf={"value": initial_value, "label": name},
|
|
)
|
|
|
|
def add_gui_number(self, name: str, initial_value: IntOrFloat) -> GuiHandle[IntOrFloat]:
|
|
"""Add a number input to the GUI.
|
|
|
|
Args:
|
|
name: The name of the number.
|
|
initial_value: The initial value of the number.
|
|
"""
|
|
assert isinstance(initial_value, (int, float))
|
|
return self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
initial_value,
|
|
leva_conf={"value": initial_value, "label": name},
|
|
)
|
|
|
|
def add_gui_vector2(
|
|
self,
|
|
name: str,
|
|
initial_value: Tuple[float, float] | onp.ndarray,
|
|
step: Optional[float] = None,
|
|
) -> GuiHandle[Tuple[float, float]]:
|
|
"""Add a length-2 vector input to the GUI.
|
|
|
|
Args:
|
|
name: The name of the vector.
|
|
initial_value: The initial value of the vector.
|
|
step: The step size for the vector.
|
|
"""
|
|
return self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
_cast_vector(initial_value, length=2),
|
|
leva_conf={
|
|
"value": initial_value,
|
|
"label": name,
|
|
"step": step,
|
|
},
|
|
)
|
|
|
|
def add_gui_vector3(
|
|
self,
|
|
name: str,
|
|
initial_value: Tuple[float, float, float] | onp.ndarray,
|
|
step: Optional[float] = None,
|
|
lock: bool = False,
|
|
) -> GuiHandle[Tuple[float, float, float]]:
|
|
"""Add a length-3 vector input to the GUI.
|
|
|
|
Args:
|
|
name: The name of the vector.
|
|
initial_value: The initial value of the vector.
|
|
step: The step size for the vector.
|
|
lock: Whether the vector is locked.
|
|
"""
|
|
return self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
_cast_vector(initial_value, length=3),
|
|
leva_conf={
|
|
"label": name,
|
|
"value": initial_value,
|
|
"step": step,
|
|
"lock": lock,
|
|
},
|
|
)
|
|
|
|
# Resolve type of value to a Literal whenever possible.
|
|
@overload
|
|
def add_gui_select(
|
|
self,
|
|
name: str,
|
|
options: List[TLiteralString],
|
|
initial_value: Optional[TLiteralString] = None,
|
|
) -> GuiSelectHandle[TLiteralString]:
|
|
...
|
|
|
|
@overload
|
|
def add_gui_select(
|
|
self,
|
|
name: str,
|
|
options: List[str],
|
|
initial_value: Optional[str] = None,
|
|
) -> GuiSelectHandle[str]:
|
|
...
|
|
|
|
def add_gui_select(
|
|
self,
|
|
name: str,
|
|
options: List[TLiteralString] | List[str],
|
|
initial_value: Optional[TLiteralString | str] = None,
|
|
) -> GuiSelectHandle[TLiteralString] | GuiSelectHandle[str]:
|
|
"""Add a dropdown to the GUI.
|
|
|
|
Args:
|
|
name: The name of the dropdown.
|
|
options: The options to choose from.
|
|
initial_value: The initial value of the dropdown.
|
|
"""
|
|
assert len(options) > 0
|
|
if initial_value is None:
|
|
initial_value = options[0]
|
|
return GuiSelectHandle(
|
|
self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
initial_value,
|
|
leva_conf={
|
|
"value": initial_value,
|
|
"label": name,
|
|
"options": options,
|
|
},
|
|
)._impl
|
|
)
|
|
|
|
def add_gui_slider(
|
|
self,
|
|
name: str,
|
|
low: IntOrFloat,
|
|
high: IntOrFloat,
|
|
step: Optional[IntOrFloat],
|
|
initial_value: IntOrFloat,
|
|
) -> GuiHandle[IntOrFloat]:
|
|
"""Add a slider to the GUI.
|
|
|
|
Args:
|
|
name: The name of the slider.
|
|
low: The minimum value of the slider.
|
|
high: The maximum value of the slider.
|
|
step: The step size of the slider.
|
|
initial_value: The initial value of the slider.
|
|
"""
|
|
assert high >= low
|
|
if step is not None:
|
|
assert step <= (high - low)
|
|
assert high >= initial_value >= low
|
|
|
|
return self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
initial_value,
|
|
leva_conf={
|
|
"value": initial_value,
|
|
"label": name,
|
|
"min": low,
|
|
"max": high,
|
|
"step": step,
|
|
},
|
|
)
|
|
|
|
def add_gui_rgb(
|
|
self,
|
|
name: str,
|
|
initial_value: Tuple[int, int, int],
|
|
) -> GuiHandle[Tuple[int, int, int]]:
|
|
"""Add an RGB picker to the GUI.
|
|
|
|
Args:
|
|
image: The image to set as the background. Must be a 3D numpy array of shape (H, W, 3).
|
|
file_format: The file format to use for the image.
|
|
quality: The quality of the image, if using jpeg. Must be an integer between 0 and 100.
|
|
"""
|
|
return self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
initial_value,
|
|
leva_conf={
|
|
"value": {
|
|
"r": initial_value[0],
|
|
"g": initial_value[1],
|
|
"b": initial_value[2],
|
|
},
|
|
"label": name,
|
|
},
|
|
encoder=lambda rgb: dict(zip("rgb", rgb)),
|
|
decoder=lambda rgb_dict: (rgb_dict["r"], rgb_dict["g"], rgb_dict["b"]),
|
|
)
|
|
|
|
def add_gui_rgba(
|
|
self,
|
|
name: str,
|
|
initial_value: Tuple[int, int, int, int],
|
|
) -> GuiHandle[Tuple[int, int, int, int]]:
|
|
"""Add an RGBA picker to the GUI."""
|
|
return self._add_gui_impl(
|
|
"/".join(self._gui_folder_labels + [name]),
|
|
initial_value,
|
|
leva_conf={
|
|
"value": {
|
|
"r": initial_value[0],
|
|
"g": initial_value[1],
|
|
"b": initial_value[2],
|
|
"a": initial_value[3],
|
|
},
|
|
"label": name,
|
|
},
|
|
encoder=lambda rgba: dict(zip("rgba", rgba)),
|
|
decoder=lambda rgba_dict: (
|
|
rgba_dict["r"],
|
|
rgba_dict["g"],
|
|
rgba_dict["b"],
|
|
rgba_dict["a"],
|
|
),
|
|
)
|
|
|
|
def set_background_image(
|
|
self,
|
|
image: onp.ndarray,
|
|
file_format: Literal["png", "jpeg"] = "jpeg",
|
|
quality: Optional[int] = None,
|
|
) -> None:
|
|
"""Set the background image of the scene.
|
|
|
|
Args:
|
|
image: The image to set as the background. Must be a 3D numpy array of shape (H, W, 3).
|
|
file_format: The file format to use for the image.
|
|
quality: The quality of the image, if using jpeg. Must be an integer between 0 and 100.
|
|
"""
|
|
media_type, base64_data = _encode_image_base64(image, file_format, quality=quality)
|
|
self._queue(messages.BackgroundImageMessage(media_type=media_type, base64_data=base64_data))
|
|
|
|
def send_file_path_info(self, config_base_dir: Path, data_base_dir: Path, export_path_name: str) -> None:
|
|
"""Send file path info to the scene.
|
|
|
|
Args:
|
|
config_base_dir: The base directory for config files.
|
|
data_base_dir: The base directory for data files.
|
|
export_path_name: The name for the export folder.
|
|
"""
|
|
self._queue(
|
|
messages.FilePathInfoMessage(
|
|
config_base_dir=str(config_base_dir),
|
|
data_base_dir=str(data_base_dir),
|
|
export_path_name=export_path_name,
|
|
)
|
|
)
|
|
|
|
def update_scene_box(self, scene_box: SceneBox) -> None:
|
|
"""Update the scene box.
|
|
|
|
Args:
|
|
scene_box: The scene box.
|
|
"""
|
|
self._queue(messages.SceneBoxMessage(min=scene_box.aabb[0].tolist(), max=scene_box.aabb[1].tolist()))
|
|
|
|
def add_dataset_image(self, idx: str, json: Dict) -> None:
|
|
"""Add a dataset image to the scene.
|
|
|
|
Args:
|
|
idx: The index of the image.
|
|
json: The json dict from the camera frustum and image.
|
|
"""
|
|
self._queue(messages.DatasetImageMessage(idx=idx, json=json))
|
|
|
|
def set_training_state(self, training_state: Literal["training", "paused", "completed"]) -> None:
|
|
"""Set the training mode.
|
|
|
|
Args:
|
|
training_state: The training mode.
|
|
"""
|
|
self._queue(messages.TrainingStateMessage(training_state=training_state))
|
|
|
|
def send_camera_paths(self, camera_paths: Dict[str, Any]) -> None:
|
|
"""Send camera paths to the scene.
|
|
|
|
Args:
|
|
camera_paths: A dictionary of camera paths.
|
|
"""
|
|
self._queue(messages.CameraPathsMessage(payload=camera_paths))
|
|
|
|
def send_crop_params(
|
|
self,
|
|
crop_enabled: bool,
|
|
crop_bg_color: Tuple[int, int, int],
|
|
crop_center: Tuple[float, float, float],
|
|
crop_scale: Tuple[float, float, float],
|
|
) -> None:
|
|
"""Send crop parameters to the scene.
|
|
|
|
Args:
|
|
crop_enabled: Whether crop is enabled.
|
|
crop_bg_color: The background color of the crop.
|
|
crop_center: The center of the crop.
|
|
crop_scale: The scale of the crop.
|
|
"""
|
|
self._queue(
|
|
messages.CropParamsMessage(
|
|
crop_enabled=crop_enabled, crop_bg_color=crop_bg_color, crop_center=crop_center, crop_scale=crop_scale
|
|
)
|
|
)
|
|
|
|
def send_status_message(self, eval_res: str, step: int):
|
|
"""Send status message
|
|
|
|
Args:
|
|
eval_res: The resolution of the render in plain text.
|
|
step: The current step.
|
|
"""
|
|
self._queue(messages.StatusMessage(eval_res=eval_res, step=step))
|
|
|
|
def send_output_options_message(self, options: List[str]):
|
|
"""Send output options message
|
|
|
|
Args:
|
|
options: The list of output options
|
|
"""
|
|
self._queue(messages.OutputOptionsMessage(options=options))
|
|
|
|
def _add_gui_impl(
|
|
self,
|
|
name: str,
|
|
initial_value: T,
|
|
leva_conf: dict,
|
|
is_button: bool = False,
|
|
encoder: Callable[[T], Any] = lambda x: x,
|
|
decoder: Callable[[Any], T] = lambda x: x,
|
|
) -> GuiHandle[T]:
|
|
"""Private helper for adding a simple GUI element."""
|
|
|
|
handle_state = _GuiHandleState(
|
|
name,
|
|
typ=type(initial_value),
|
|
api=self,
|
|
value=initial_value,
|
|
last_updated=time.time(),
|
|
folder_labels=self._gui_folder_labels,
|
|
update_cb=[],
|
|
leva_conf=leva_conf,
|
|
is_button=is_button,
|
|
encoder=encoder,
|
|
decoder=decoder,
|
|
)
|
|
self._handle_state_from_gui_name[name] = handle_state
|
|
handle_state.cleanup_cb = lambda: self._handle_state_from_gui_name.pop(name)
|
|
|
|
# For broadcasted GUI handles, we should synchronize all clients.
|
|
# This will be a no-op for client handles.
|
|
if not is_button:
|
|
|
|
def sync_other_clients(client_id: ClientId, value: Any) -> None:
|
|
message = messages.GuiSetValueMessage(name=name, value=handle_state.encoder(value))
|
|
message.excluded_self_client = client_id
|
|
self._queue(message)
|
|
|
|
handle_state.sync_cb = sync_other_clients
|
|
|
|
self._queue(
|
|
messages.GuiAddMessage(
|
|
name=name,
|
|
folder_labels=tuple(self._gui_folder_labels),
|
|
leva_conf=leva_conf,
|
|
)
|
|
)
|
|
return GuiHandle(handle_state)
|
|
|
|
def use_time_conditioning(self) -> None:
|
|
"""Use time conditioning."""
|
|
self._queue(messages.UseTimeConditioningMessage())
|
|
|
|
def _handle_gui_updates(
|
|
self: MessageApi,
|
|
client_id: ClientId,
|
|
message: messages.GuiUpdateMessage,
|
|
) -> None:
|
|
handle_state = self._handle_state_from_gui_name.get(message.name, None)
|
|
if handle_state is None:
|
|
return
|
|
|
|
value = handle_state.typ(handle_state.decoder(message.value))
|
|
|
|
# Only call update when value has actually changed.
|
|
if not handle_state.is_button and value == handle_state.value:
|
|
return
|
|
|
|
# Update state.
|
|
handle_state.value = value
|
|
handle_state.last_updated = time.time()
|
|
|
|
# Trigger callbacks.
|
|
for cb in handle_state.update_cb:
|
|
cb(GuiHandle(handle_state))
|
|
if handle_state.sync_cb is not None:
|
|
handle_state.sync_cb(client_id, value)
|