diff --git a/.vscode/settings.json b/.vscode/settings.json index 5e3f61611..90f5ae01c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -121,5 +121,6 @@ "__node_handle": "cpp" }, "python.analysis.typeCheckingMode": "basic", - "python.analysis.diagnosticMode": "workspace" + "python.analysis.diagnosticMode": "workspace", + "eslint.packageManager": "yarn" } diff --git a/docs/developer_guides/viewer/custom_gui.md b/docs/developer_guides/viewer/custom_gui.md new file mode 100644 index 000000000..7ddd11a61 --- /dev/null +++ b/docs/developer_guides/viewer/custom_gui.md @@ -0,0 +1,106 @@ +# Custom GUI + +We provide support for custom viewer GUI elements that can be defined in any `nn.Module`. Although we don't have any specific use cases in mind, here are some examples of what can be achieved with this feature: + +- Using text input to modify the rendering +- Logging numerical values to the viewer +- Using checkboxes to turn off and on losses +- Using a dropdown to switch between appearances + +## Adding an Element + +To define a custom element, create an instance of one of the provided classes in `nerfstudio.viewer.server.viewer_elements`, and assign it as a class variable in your `nn.Module`. + +```python +from nerfstudio.viewer.server.viewer_elements import ViewerNumber + +class MyClass(nn.Module):#must inherit from nn.Module + def __init__(self): + # Must be a class variable + self.custom_value = ViewerNumber(name="My Value", default_value=1.0) +``` +**Element Hierarchy** +The viewer recursively searches all `nn.Module` children of the base `Pipeline` object, and arranges parameters into folders based on their variable names. +For example, a `ViewerElement` defined in `pipeline.model.field` will be in the "Custom/model/field" folder in the GUI. + +**Reading the value** +To read the value of a custom element, simply access its `value` attribute. In this case it will be `1.0` unless modified by the user in the viewer. + +```python +current_value = self.custom_value.value +``` + +**Callbacks** +You can register a callback that will be called whenever a new value for your GUI element is available. For example, one can use a callback to update config parameters when elements are changed: +```python +def on_change_callback(handle: GuiHandle): + self.config.example_parameter = handle.get_value() + +self.custom_checkbox = ViewerCheckbox( + name="Checkbox", + default_value = False, + cb_hook = on_change_callback +) +``` +The GuiHandle value is a viser object which you can use to get the new value. + +**Thread safety** +Note that `ViewerElement` values can change asynchronously to model execution. So, it's best practice to store the value of a viewer element once at the beginning +of a forward pass and refer to the static variable afterwards. +```python +class MyModel(Model): + def __init__(self): + self.slider = ViewerSlider(name="Slider", default_value=0.5, min_value=0.0, max_value=1.0) + + def get_outputs(self,ray) + slider_val = self.slider.value + #self.slider.value could change after this, unsafe to use + +``` + + +**Writing to the element** +You can write to a viewer element in Python, which provides a convenient way to track values in your code without the need for wandb/tensorboard or relying on `print` statements. + +```python +self.custom_value = x +``` + +:::{admonition} Warning +:class: warning + +Note that updating module state while training can have unexpected side effects. It is up to the user to ensure that GUI actions are safe. In most cases, we recommend wrapping GUI related logic in `if self.training:` to ensure that the effects are only applied during inference. +::: + +## Example Elements + +```{image} imgs/custom_controls.png +:align: center +:width: 400 +``` + +This was created with the following elements: + +```python +from nerfstudio.viewer.server.viewer_elements import * + +class MyModel(Model): + def __init__(self): + self.a = ViewerButton(name="My Button", call_fn=self.handle_btn) + self.b = ViewerNumber(name="Number", default_value=1.0) + self.c = ViewerCheckbox(name="Checkbox", default_value=False) + self.d = ViewerDropdown(name="Dropdown", default_value="A", options=["A", "B"]) + self.e = ViewerSlider(name="Slider", default_value=0.5, min_value=0.0, max_value=1.0) + self.f = ViewerText(name="Text", default_value="Hello World") + self.g = ViewerVec3(name="3D Vector", default_value=(0.1, 0.7, 0.1)) + + self.rgb_renderer = RGBRenderer() +... +class RGBRenderer(nn.Module): + def __init__(self): + #lives in "Custom/model/rgb_renderer" GUI folder + self.a = ViewerRGB(name="F", default_value=(0.1, 0.7, 0.1)) +... +``` + +For more information on the available classes and their arguments, refer to the [API documentation](../../reference/api/viewer.rst) diff --git a/docs/developer_guides/viewer/imgs/custom_controls.png b/docs/developer_guides/viewer/imgs/custom_controls.png new file mode 100644 index 000000000..557f76b5a Binary files /dev/null and b/docs/developer_guides/viewer/imgs/custom_controls.png differ diff --git a/docs/developer_guides/viewer/imgs/viewer_dark.png b/docs/developer_guides/viewer/imgs/viewer_dark.png deleted file mode 100644 index d2abe9a10..000000000 Binary files a/docs/developer_guides/viewer/imgs/viewer_dark.png and /dev/null differ diff --git a/docs/developer_guides/viewer/imgs/viewer_figure.png b/docs/developer_guides/viewer/imgs/viewer_figure.png deleted file mode 100644 index 3d0fd2846..000000000 Binary files a/docs/developer_guides/viewer/imgs/viewer_figure.png and /dev/null differ diff --git a/docs/developer_guides/viewer/imgs/viewer_light.png b/docs/developer_guides/viewer/imgs/viewer_light.png deleted file mode 100644 index 2220e3a8c..000000000 Binary files a/docs/developer_guides/viewer/imgs/viewer_light.png and /dev/null differ diff --git a/docs/developer_guides/viewer/index.md b/docs/developer_guides/viewer/index.md new file mode 100644 index 000000000..eebc29d32 --- /dev/null +++ b/docs/developer_guides/viewer/index.md @@ -0,0 +1,22 @@ +# Viewer + +> We have a real-time web viewer that requires no installation. It's available at [https://viewer.nerf.studio/](https://viewer.nerf.studio/), where you can connect to your training job. + +The viewer is built on [Viser](https://github.com/brentyi/viser/tree/main/viser) using [ThreeJS](https://threejs.org/) and packaged into a [ReactJS](https://reactjs.org/) application. This client viewer application will connect via a websocket to a server running on your machine. + +```{toctree} +:titlesonly: + +custom_gui +local_viewer +``` + +## Acknowledgements and references + +We thank the authors and contributors to the following repos, which we've started, used, and modified for our use-cases. + +- [Viser](https://github.com/brentyi/viser/tree/main/viser) - made by [Brent Yi](https://github.com/brentyi) +- [meshcat-python](https://github.com/rdeits/meshcat-python) - made by [Robin Deits](https://github.com/rdeits) +- [meshcat](https://github.com/rdeits/meshcat) - made by [Robin Deits](https://github.com/rdeits) +- [ThreeJS](https://threejs.org/) +- [ReactJS](https://reactjs.org/) diff --git a/docs/developer_guides/viewer/local_viewer.md b/docs/developer_guides/viewer/local_viewer.md new file mode 100644 index 000000000..d5912bd89 --- /dev/null +++ b/docs/developer_guides/viewer/local_viewer.md @@ -0,0 +1,62 @@ +# Local Server + +If you are unable to connect to `https://viewer.nerf.studio`, want to use Safari, or want develop the viewer codebase, you can launch your own local viewer. + +## Installing Dependencies + +```shell +cd nerfstudio/viewer/app +``` + +Install npm (to install yarn) and yarn + +```shell +sudo apt-get install npm +npm install --global yarn +``` + +Install nvm and set the node version +Install nvm with [instructions](https://heynode.com/tutorial/install-nodejs-locally-nvm/). + +```shell +nvm install 17.8.0 +``` + +Now running `node --version` in the shell should print "v17.8.0". +Install package.json dependencies and start the client viewer app: + +```shell +yarn install +``` + +## Launch the web client + +From the `nerfstudio/viewer/app` folder, run: + +```shell +yarn start +``` + +The local webserver runs on port 4000 by default, +so when `ns-train` is running, you can connect to the viewer locally at +[http://localhost:4000/?websocket_url=ws://localhost:7007](http://localhost:4000/?websocket_url=ws://localhost:7007) + +## FAQ + +### Engine node incompatible + +While running `yarn install`, you run into: `The engine "node" is incompatible with this module.` + +**Solution**: + +Install nvm with instructions at [instructions](https://heynode.com/tutorial/install-nodejs-locally-nvm/). + +```shell +nvm install 17.8.0 +``` + +If you cannot install nvm, try ignoring the engines + +```shell +yarn install --ignore-engines +``` diff --git a/docs/developer_guides/viewer/viewer_overview.md b/docs/developer_guides/viewer/viewer_overview.md deleted file mode 100644 index 55dbab2f2..000000000 --- a/docs/developer_guides/viewer/viewer_overview.md +++ /dev/null @@ -1,98 +0,0 @@ -# Viewer overview - -> We have a real-time web viewer that requires no installation. It's available at [https://viewer.nerf.studio/](https://viewer.nerf.studio/), where you can connect to your training job. If you want to understand how the viewer works and contribute to it's features, this section is for you! - -The viewer is built using [ThreeJS](https://threejs.org/) and packaged into a [ReactJS](https://reactjs.org/) application. This client viewer application will connect via a websocket to a server running on your machine. The following figure helps to illustrate how our viewer framework works: - -```{image} imgs/viewer_light.png -:align: center -:class: only-light -``` - -```{image} imgs/viewer_dark.png -:align: center -:class: only-dark -``` - -## System design - -#### Connection between nerfstudio and the Bridge Server - -In the center, we have the Bridge Server, which facilitates the connection between nerfstudio code and the Client App. This server runs on the same machine that you are using nerfstudio. It has a TCP Request/Reply (REQ/REP) connection that nerfstudio can connect to with the Viewer object (left). We use [ZeroMQ](https://zeromq.org/), an open-sourced messaging library, to implement this lightweight TCP connection. The Viewer class can send commands to the Bridge Server and receive replies. The Bridge Server will either dispatch commands to the Client App via a websocket or it will return information stored in the Bridge Server state. - -#### Connection between the Bridge Server and the Client App - -The connection between the Bridge Server and the Client App works with WebSockets. - -- **WebSocket connection** - The WebSocket is used by the Bridge Server to dispatch commands coming from the nerfstudio TCP connection. Commands can be used for drawing primitives, for setting the transform of objects, for the setting various properties, and more. - -## Installing and running locally - -#### Running the Client App - -```shell -cd nerfstudio/viewer/app -``` - -Install npm (to install yarn) and yarn - -```shell -sudo apt-get install npm -npm install --global yarn -``` - -Install nvm and set the node version -Install nvm with [instructions](https://heynode.com/tutorial/install-nodejs-locally-nvm/). - -```shell -nvm install 17.8.0 -``` - -Now running `node --version` in the shell should print "v17.8.0". -Install package.json dependencies and start the client viewer app - -```shell -yarn install -yarn start -``` - -The local webserver runs on port 4000 by default, -so when `ns-train` is running, you can connect to the viewer locally at -[http://localhost:4000/?websocket_url=ws://localhost:7007](http://localhost:4000/?websocket_url=ws://localhost:7007) - -#### Running the Bridge Server Manually - -The viewer bridge server runs on the same machine that you use for training. The training code will connect to this server with a lightweight TCP connection using ZMQ. The training job will launch the viewer bridge server is you specify `--viewer.launch-bridge-server` in the terminal. Otherwise, you can launch the bridge server manually with the following script. - -```python -ns-bridge-server --help -``` - -## Acknowledgements and references - -We thank the authors and contributors to the following repos, which we've started, used, and modified for our use-cases. - -- [meshcat-python](https://github.com/rdeits/meshcat-python) - made by [Robin Deits](https://github.com/rdeits) -- [meshcat](https://github.com/rdeits/meshcat) - made by [Robin Deits](https://github.com/rdeits) -- [ThreeJS](https://threejs.org/) -- [ReactJS](https://reactjs.org/) - -## FAQ - -#### Engine node incompatible - -While running `yarn install`, you run into: `The engine "node" is incompatible with this module.` - -**Solution**: - -Install nvm with instructions at [instructions](https://heynode.com/tutorial/install-nodejs-locally-nvm/). - -```shell -nvm install 17.8.0 -``` - -If you cannot install nvm, try ignoring the engines - -``` -yarn install --ignore-engines -``` diff --git a/docs/index.md b/docs/index.md index d28393883..ee611c573 100644 --- a/docs/index.md +++ b/docs/index.md @@ -77,7 +77,7 @@ nerfology/model_components/index developer_guides/new_methods developer_guides/pipelines/index -developer_guides/viewer/viewer_overview +developer_guides/viewer/index developer_guides/config developer_guides/debugging_tools/index ``` diff --git a/docs/reference/api/viewer.rst b/docs/reference/api/viewer.rst index 508fddf68..4be105c86 100644 --- a/docs/reference/api/viewer.rst +++ b/docs/reference/api/viewer.rst @@ -3,6 +3,6 @@ Viewer ============ -.. automodule:: nerfstudio.viewer +.. automodule:: nerfstudio.viewer.server.viewer_elements :members: :show-inheritance: diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index 3d7872d74..fd5608b4e 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -131,19 +131,10 @@ class ViewerConfig(PrintableConfig): relative_log_filename: str = "viewer_log_filename.txt" """Filename to use for the log file.""" - start_train: bool = True - """whether to immediately start training upon loading viewer - if False, will just visualize dataset but you can toggle training in viewer""" - zmq_port: Optional[int] = None - """The zmq port to connect to for communication. If None, find an available port.""" - launch_bridge_server: bool = True - """whether or not to launch the bridge server""" websocket_port: Optional[int] = None """The websocket port to connect to. If None, find an available port.""" websocket_port_default: int = 7007 """The default websocket port to connect to if websocket_port is not specified""" - ip_address: str = "127.0.0.1" - """the ip address where the bridge server is running""" num_rays_per_chunk: int = 32768 """number of rays per chunk to render with viewer""" max_num_display_images: int = 512 @@ -155,5 +146,3 @@ class ViewerConfig(PrintableConfig): """Image format viewer should use; jpeg is lossy compression, while png is lossless.""" jpeg_quality: int = 90 """Quality tradeoff to use for jpeg compression.""" - png_compression: int = 1 - """Size/speed tradeoff to use for png compression.""" diff --git a/nerfstudio/data/scene_box.py b/nerfstudio/data/scene_box.py index e28761e29..e513a1428 100644 --- a/nerfstudio/data/scene_box.py +++ b/nerfstudio/data/scene_box.py @@ -17,7 +17,7 @@ Dataset input structures. """ from dataclasses import dataclass -from typing import Dict, Union +from typing import Union import torch from torchtyping import TensorType @@ -64,21 +64,6 @@ class SceneBox: normalized_positions = (positions - aabb[0]) / aabb_lengths return normalized_positions - def to_json(self) -> Dict: - """Returns a json object from the Python object.""" - return {"type": "aabb", "min_point": self.aabb[0].tolist(), "max_point": self.aabb[1].tolist()} - - @staticmethod - def from_json(json_: Dict) -> "SceneBox": - """Returns the an instance of SceneBox from a json dictionary. - - Args: - json_: the json dictionary containing scene box information - """ - assert json_["type"] == "aabb" - aabb = torch.tensor([json_[0], json_[1]]) - return SceneBox(aabb=aabb) - @staticmethod def from_camera_poses(poses: TensorType[..., 3, 4], scale_factor: float) -> "SceneBox": """Returns the instance of SceneBox that fully envelopes a set of poses diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 275814492..7f07a3b5d 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -23,6 +23,7 @@ import os import time from dataclasses import dataclass, field from pathlib import Path +from threading import Lock from typing import Dict, List, Optional, Tuple, Type, Union import torch @@ -46,7 +47,7 @@ from nerfstudio.utils.decorators import ( ) from nerfstudio.utils.misc import step_check from nerfstudio.utils.writer import EventName, TimeWriter -from nerfstudio.viewer.server import viewer_utils +from nerfstudio.viewer.server.viewer_state import ViewerState CONSOLE = Console(width=120) @@ -103,6 +104,7 @@ class Trainer: pipeline: The pipeline object. optimizers: The optimizers object. callbacks: The callbacks object. + is_training: Whether the model is training. """ pipeline: VanillaPipeline @@ -110,11 +112,13 @@ class Trainer: callbacks: List[TrainingCallback] def __init__(self, config: TrainerConfig, local_rank: int = 0, world_size: int = 1) -> None: + self.train_lock = Lock() self.config = config self.local_rank = local_rank self.world_size = world_size self.device: TORCH_DEVICE = "cpu" if world_size == 0 else f"cuda:{local_rank}" self.mixed_precision: bool = self.config.mixed_precision + self.is_training: bool = True if self.device == "cpu": self.mixed_precision = False CONSOLE.print("Mixed precision is disabled for CPU training.") @@ -138,6 +142,11 @@ class Trainer: 'test': loads train/test datasets into memory 'inference': does not load any dataset into memory """ + self.pipeline = self.config.pipeline.setup( + device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank + ) + self.optimizers = self.setup_optimizers() + # set up viewer if enabled viewer_log_path = self.base_dir / self.config.viewer.relative_log_filename self.viewer_state, banner_messages = None, None @@ -145,16 +154,17 @@ class Trainer: datapath = self.config.data if datapath is None: datapath = self.base_dir - self.viewer_state, banner_messages = viewer_utils.setup_viewer( - self.config.viewer, log_filename=viewer_log_path, datapath=datapath + self.viewer_state = ViewerState( + self.config.viewer, + log_filename=viewer_log_path, + datapath=datapath, + pipeline=self.pipeline, + trainer=self, + train_lock=self.train_lock, ) + banner_messages = [f"Viewer at: {self.viewer_state.viewer_url}"] self._check_viewer_warnings() - self.pipeline = self.config.pipeline.setup( - device=self.device, test_mode=test_mode, world_size=self.world_size, local_rank=self.local_rank - ) - self.optimizers = self.setup_optimizers() - self._load_checkpoint() self.callbacks = self.pipeline.get_training_callbacks( @@ -206,21 +216,26 @@ class Trainer: num_iterations = self.config.max_num_iterations step = 0 for step in range(self._start_step, self._start_step + num_iterations): - with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t: - self.pipeline.train() + while not self.is_training: + time.sleep(0.01) + with self.train_lock: + with TimeWriter(writer, EventName.ITER_TRAIN_TIME, step=step) as train_t: + self.pipeline.train() - # training callbacks before the training iteration - for callback in self.callbacks: - callback.run_callback_at_location( - step, location=TrainingCallbackLocation.BEFORE_TRAIN_ITERATION - ) + # training callbacks before the training iteration + for callback in self.callbacks: + callback.run_callback_at_location( + step, location=TrainingCallbackLocation.BEFORE_TRAIN_ITERATION + ) - # time the forward pass - loss, loss_dict, metrics_dict = self.train_iteration(step) + # time the forward pass + loss, loss_dict, metrics_dict = self.train_iteration(step) - # training callbacks after the training iteration - for callback in self.callbacks: - callback.run_callback_at_location(step, location=TrainingCallbackLocation.AFTER_TRAIN_ITERATION) + # training callbacks after the training iteration + for callback in self.callbacks: + callback.run_callback_at_location( + step, location=TrainingCallbackLocation.AFTER_TRAIN_ITERATION + ) # Skip the first two steps to avoid skewed timings that break the viewer rendering speed estimate. if step > 1: @@ -266,14 +281,8 @@ class Trainer: CONSOLE.print("[bold green]:tada: :tada: :tada: Training Finished :tada: :tada: :tada:", justify="center") if not self.config.viewer.quit_on_train_completion: CONSOLE.print("Use ctrl+c to quit", justify="center") - self._always_render(step) - - @check_main_thread - def _always_render(self, step: int) -> None: - if self.viewer_state is not None: while True: - self.viewer_state.vis["renderingState/isTraining"].write(False) - self._update_viewer_state(step) + time.sleep(0.01) @check_main_thread def _check_viewer_warnings(self) -> None: @@ -295,10 +304,8 @@ class Trainer: assert self.viewer_state and self.pipeline.datamanager.train_dataset self.viewer_state.init_scene( dataset=self.pipeline.datamanager.train_dataset, - start_train=self.config.viewer.start_train, + start_train=True, ) - if not self.config.viewer.start_train: - self._always_render(self._start_step) @check_viewer_enabled def _update_viewer_state(self, step: int) -> None: @@ -309,16 +316,12 @@ class Trainer: step: current train step """ assert self.viewer_state is not None - with TimeWriter(writer, EventName.ITER_VIS_TIME, step=step) as _: - num_rays_per_batch: int = self.pipeline.datamanager.get_train_rays_per_batch() - try: - self.viewer_state.update_scene(self, step, self.pipeline.model, num_rays_per_batch) - except RuntimeError: - time.sleep(0.03) # sleep to allow buffer to reset - assert self.viewer_state.vis is not None - self.viewer_state.vis["renderingState/log_errors"].write( - "Error: GPU out of memory. Reduce resolution to prevent viewer from crashing." - ) + num_rays_per_batch: int = self.pipeline.datamanager.get_train_rays_per_batch() + try: + self.viewer_state.update_scene(step, num_rays_per_batch) + except RuntimeError: + time.sleep(0.03) # sleep to allow buffer to reset + CONSOLE.log("Viewer failed. Continuing training.") @check_viewer_enabled def _update_viewer_rays_per_sec(self, train_t: TimeWriter, vis_t: TimeWriter, step: int) -> None: @@ -395,8 +398,10 @@ class Trainer: Args: step: Current training step. """ + self.optimizers.zero_grad_all() cpu_or_cuda_str: str = self.device.split(":")[0] + with torch.autocast(device_type=cpu_or_cuda_str, enabled=self.mixed_precision): _, loss_dict, metrics_dict = self.pipeline.get_train_loss_dict(step=step) loss = functools.reduce(torch.add, loss_dict.values()) diff --git a/nerfstudio/utils/eval_utils.py b/nerfstudio/utils/eval_utils.py index 3431b20a3..bd535af7f 100644 --- a/nerfstudio/utils/eval_utils.py +++ b/nerfstudio/utils/eval_utils.py @@ -33,13 +33,15 @@ from nerfstudio.pipelines.base_pipeline import Pipeline CONSOLE = Console(width=120) -def eval_load_checkpoint(config: TrainerConfig, pipeline: Pipeline) -> Path: +def eval_load_checkpoint(config: TrainerConfig, pipeline: Pipeline) -> Tuple[Path, int]: ## TODO: ideally eventually want to get this to be the same as whatever is used to load train checkpoint too """Helper function to load checkpointed pipeline Args: config (DictConfig): Configuration of pipeline to load pipeline (Pipeline): Pipeline instance of which to load weights + Returns: + A tuple of the path to the loaded checkpoint and the step at which it was saved. """ assert config.load_dir is not None if config.load_step is None: @@ -61,14 +63,14 @@ def eval_load_checkpoint(config: TrainerConfig, pipeline: Pipeline) -> Path: loaded_state = torch.load(load_path, map_location="cpu") pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"]) CONSOLE.print(f":white_check_mark: Done loading checkpoint from {load_path}") - return load_path + return load_path, load_step def eval_setup( config_path: Path, eval_num_rays_per_chunk: Optional[int] = None, test_mode: Literal["test", "val", "inference"] = "test", -) -> Tuple[TrainerConfig, Pipeline, Path]: +) -> Tuple[TrainerConfig, Pipeline, Path, int]: """Shared setup for loading a saved pipeline for evaluation. Args: @@ -81,7 +83,7 @@ def eval_setup( Returns: - Loaded config, pipeline module, and corresponding checkpoint. + Loaded config, pipeline module, corresponding checkpoint, and step """ # load save config config = yaml.load(config_path.read_text(), Loader=yaml.Loader) @@ -102,6 +104,6 @@ def eval_setup( pipeline.eval() # load checkpointed information - checkpoint_path = eval_load_checkpoint(config, pipeline) + checkpoint_path, step = eval_load_checkpoint(config, pipeline) - return config, pipeline, checkpoint_path + return config, pipeline, checkpoint_path, step diff --git a/nerfstudio/utils/writer.py b/nerfstudio/utils/writer.py index e133d3757..4d168d508 100644 --- a/nerfstudio/utils/writer.py +++ b/nerfstudio/utils/writer.py @@ -47,7 +47,6 @@ class EventName(enum.Enum): ITER_TRAIN_TIME = "Train Iter (time)" TOTAL_TRAIN_TIME = "Train Total (time)" - ITER_VIS_TIME = "Viewer Rendering (time)" ETA = "ETA (time)" TRAIN_RAYS_PER_SEC = "Train Rays / Sec" TEST_RAYS_PER_SEC = "Test Rays / Sec" diff --git a/nerfstudio/viewer/app/.eslintrc.json b/nerfstudio/viewer/app/.eslintrc.json index ecb608b57..f219c3898 100644 --- a/nerfstudio/viewer/app/.eslintrc.json +++ b/nerfstudio/viewer/app/.eslintrc.json @@ -4,10 +4,12 @@ "es2021": true }, "extends": [ + "react-app", "eslint:recommended", "plugin:react/recommended", "airbnb", - "prettier" + "prettier", + "plugin:import/typescript" ], "parser": "@typescript-eslint/parser", "parserOptions": { @@ -27,6 +29,20 @@ "prefer-destructuring": "off", "react/destructuring-assignment": "off", "react/prop-types": 0, - "unused-imports/no-unused-imports-ts": 2 + "react/jsx-filename-extension": [ + 2, + { "extensions": [".js", ".jsx", ".ts", ".tsx"] } + ], + "unused-imports/no-unused-imports-ts": 2, + "import/extensions": [ + "error", + "ignorePackages", + { + "js": "never", + "jsx": "never", + "ts": "never", + "tsx": "never" + } + ] } } diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index 3525ebd86..349b25249 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-03-9-0", + "version": "23-04-10-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", @@ -15,6 +15,7 @@ "@testing-library/jest-dom": "^5.16.4", "@testing-library/react": "^13.3.0", "@testing-library/user-event": "^14.2.0", + "await-lock": "^2.2.2", "camera-controls": "^1.37.2", "classnames": "^2.3.1", "dat.gui": "^0.7.9", @@ -22,8 +23,10 @@ "eslint-config-prettier": "^8.5.0", "eslint-plugin-unused-imports": "^2.0.0", "leva": "^0.9.29", + "lodash": "^4.17.21", "meshline": "^2.0.4", "msgpack-lite": "^0.1.26", + "msgpackr": "^1.8.5", "prop-types": "^15.8.1", "re-resizable": "^6.9.9", "react": "^18.1.0", @@ -77,7 +80,7 @@ "eslint-plugin-react-hooks": "^4.3.0", "prettier": "2.7.1", "react-scripts": "^5.0.1", - "typescript": "^4.7.3", + "typescript": "^5.0.3", "wait-on": "^6.0.1" }, "resolutions": { diff --git a/nerfstudio/viewer/app/src/App.jsx b/nerfstudio/viewer/app/src/App.jsx index 1d2987107..fd42a37f9 100644 --- a/nerfstudio/viewer/app/src/App.jsx +++ b/nerfstudio/viewer/app/src/App.jsx @@ -1,14 +1,11 @@ import { CssBaseline, ThemeProvider } from '@mui/material'; import React from 'react'; -import { - SceneTreeWebSocketListener, - get_scene_tree, -} from './modules/Scene/Scene'; +import { get_scene_tree } from './modules/Scene/Scene'; import Banner from './modules/Banner'; import { BasicTabs } from './modules/SidePanel/SidePanel'; import ViewerWindow from './modules/ViewerWindow/ViewerWindow'; -import { appTheme } from './themes/theme.ts'; +import { appTheme } from './themes/theme'; export default function App() { // The scene tree won't rerender but it will listen to changes @@ -20,8 +17,6 @@ export default function App() {
- {/* Listens for websocket 'write' messages and updates the redux store. */} - {/* The banner at the top of the page. */}
diff --git a/nerfstudio/viewer/app/src/index.jsx b/nerfstudio/viewer/app/src/index.jsx index 2d16f4a01..b712d1d3d 100644 --- a/nerfstudio/viewer/app/src/index.jsx +++ b/nerfstudio/viewer/app/src/index.jsx @@ -3,14 +3,14 @@ import React from 'react'; import { createRoot } from 'react-dom/client'; import { Provider } from 'react-redux'; import App from './App'; -import WebSocketProvider from './modules/WebSocket/WebSocket'; import store from './store'; +import { ViserWebSocket } from './modules/WebSocket/ViserWebSocket'; const root = createRoot(document.getElementById('root')); root.render( - + - + , ); diff --git a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx index 5b20b545e..60460339d 100644 --- a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx @@ -1,345 +1,152 @@ import * as React from 'react'; -import { Leva, buttonGroup, useControls } from 'leva'; -import { useContext, useEffect, useState } from 'react'; +import { + LevaPanel, + useCreateStore, + Leva, + useControls, + folder, + button, +} from 'leva'; import { useDispatch, useSelector } from 'react-redux'; +import { Box } from '@mui/material'; import LevaTheme from '../../themes/leva_theme.json'; +import { + ViserWebSocketContext, + sendWebsocketMessage, + makeThrottledMessageSender, +} from '../WebSocket/ViserWebSocket'; -import { WebSocketContext } from '../WebSocket/WebSocket'; +function CustomLeva() { + const viser_websocket = React.useContext(ViserWebSocketContext); + const customGui = useSelector((state) => state.custom_gui); + const guiNames = customGui.guiNames; + const guiConfigFromName = customGui.guiConfigFromName; -const msgpack = require('msgpack-lite'); - -function dispatch_and_send(websocket, dispatch, path, data) { - dispatch({ - type: 'write', - path, - data, - }); - if (websocket.readyState === WebSocket.OPEN) { - const message = msgpack.encode({ - type: 'write', - path, - data, + // We're going to try and build an object that looks like: + // {"folder name": {"input name": leva config}} + const guiConfigTree: { [key: string]: any } = {}; + function getFolderContainer(folderLabels: string[]) { + let guiConfigNode = guiConfigTree; + folderLabels.forEach((label) => { + if (guiConfigNode[label] === undefined) { + guiConfigNode[label] = { _is_folder_marker: true }; + } + guiConfigNode = guiConfigNode[label]; }); - websocket.send(message); + return guiConfigNode; } -} -function ControlsLeva() { - // connection status indicators - const websocket = useContext(WebSocketContext).socket; - const outputOptions = useSelector( - (state) => state.renderingState.output_options, - ); - const outputChoice = useSelector( - (state) => state.renderingState.output_choice, - ); - const colormapOptions = useSelector( - (state) => state.renderingState.colormap_options, - ); - const colormapChoice = useSelector( - (state) => state.renderingState.colormap_choice, - ); - const colormapInvert = useSelector( - (state) => state.renderingState.colormap_invert, - ); - const colormapNormalize = useSelector( - (state) => state.renderingState.colormap_normalize, - ); - const max_resolution = useSelector( - (state) => state.renderingState.maxResolution, - ); - const target_train_util = useSelector( - (state) => state.renderingState.targetTrainUtil, - ); - const render_time = useSelector((state) => state.renderingState.renderTime); - const crop_enabled = useSelector( - (state) => state.renderingState.crop_enabled, - ); + guiNames.forEach((key) => { + const { levaConf, folderLabels, hidden } = guiConfigFromName[key]; + if (hidden) return; + const leafFolder = getFolderContainer(folderLabels); - const crop_bg_color = useSelector( - (state) => state.renderingState.crop_bg_color, + // Hacky stuff that lives outside of TypeScript... + if (levaConf.type === 'BUTTON') { + // Add a button. + leafFolder[key] = button(() => { + sendWebsocketMessage(viser_websocket, { + type: 'GuiUpdateMessage', + name: key, + value: true, + }); + }, levaConf.settings); + } else { + // Add any other kind of input. + const throttledSender = makeThrottledMessageSender(viser_websocket, 25); + leafFolder[key] = { + ...levaConf, + onChange: (value: any, _propName: any, options: any) => { + if (options.initial) return; + throttledSender({ + type: 'GuiUpdateMessage', + name: key, + value, + }); + }, + }; + } + }); + + // Recursively wrap folders in a GUI config tree with Leva's `folder()`. + function wrapFoldersInGuiConfigTree( + guiConfigNode: { [key: string]: any }, + root: boolean, + ) { + const { _is_folder_marker, ...rest } = guiConfigNode; + const nodeCopy = { ...rest }; + + if (root || _is_folder_marker === true) { + const out: { [key: string]: any } = {}; + Object.entries(nodeCopy).forEach(([k, v]) => { + out[k] = wrapFoldersInGuiConfigTree(v, false); + }); + return root ? out : folder(out); + } + return guiConfigNode; + } + + // Make Leva controls. + const levaStore = useCreateStore(); + const [, set] = useControls( + () => wrapFoldersInGuiConfigTree(guiConfigTree, true), + { store: levaStore }, + [guiConfigFromName], ); - const crop_scale = useSelector((state) => state.renderingState.crop_scale); - - const crop_center = useSelector((state) => state.renderingState.crop_center); - const dispatch = useDispatch(); + const guiSetQueue = customGui.guiSetQueue; + React.useEffect(() => { + // This line is important to prevent looping + if (Object.keys(guiSetQueue).length === 0) return; + Object.entries(guiSetQueue).forEach(([key, value]) => { + if (guiNames.includes(key)) { + set({ [key]: value }); // call the leva function for setting the value of the element + } + }); + // delete the queue + dispatch({ + type: 'write', + path: 'custom_gui/guiSetQueue', + data: {}, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [guiSetQueue, set, guiNames]); - const [display_render_time, set_display_render_time] = useState(false); - - const receive_temporal_dist = (e) => { - const msg = msgpack.decode(new Uint8Array(e.data)); - if (msg.path === '/model/has_temporal_distortion') { - set_display_render_time(msg.data === 'true'); - websocket.removeEventListener('message', receive_temporal_dist); - } - }; - websocket.addEventListener('message', receive_temporal_dist); - - const [, setControls] = useControls( - () => ({ - // training speed - SpeedButtonGroup: buttonGroup({ - label: `Train Speed`, - hint: 'Select the training speed, affects viewer render quality, not final render quality', - opts: { - Fast: () => - setControls({ target_train_util: 0.9, max_resolution: 512 }), - Balanced: () => - setControls({ target_train_util: 0.7, max_resolution: 1024 }), - Slow: () => - setControls({ target_train_util: 0.1, max_resolution: 2048 }), - }, - }), - // output_options - output_options: { - label: 'Output Render', - options: [...new Set(outputOptions)], - value: outputChoice, - hint: 'Select the output to render', - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/output_choice', - v, - ); - }, - }, - // colormap_options - colormap_options: { - label: 'Colormap', - options: colormapOptions, - value: colormapChoice, - hint: 'Select the colormap to use', - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/colormap_choice', - v, - ); - }, - disabled: colormapOptions.length === 1, - }, - colormap_invert: { - label: '| Invert', - value: colormapInvert, - hint: 'Invert the colormap', - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/colormap_invert', - v, - ); - }, - render: (get) => get('colormap_options') !== 'default', - }, - colormap_normalize: { - label: '| Normalize', - value: colormapNormalize, - hint: 'Whether to normalize output between 0 and 1', - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/colormap_normalize', - v, - ); - }, - render: (get) => get('colormap_options') !== 'default', - }, - colormap_range: { - label: '| Range', - value: [0, 1], - step: 0.01, - min: -2, - max: 5, - hint: 'Min and max values of the colormap', - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/colormap_range', - v, - ); - }, - render: (get) => get('colormap_options') !== 'default', - }, - // Dynamic Resolution - target_train_util: { - label: 'Train Util.', - value: target_train_util, - min: 0, - max: 1, - step: 0.05, - hint: "Target training utilization, 0.0 is slow, 1.0 is fast, doesn't affect final render quality", - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/targetTrainUtil', - v, - ); - }, - }, - // resolution - max_resolution: { - label: 'Max Res.', - value: max_resolution, - min: 256, - max: 2048, - step: 1, - hint: 'Maximum resolution to render in viewport', - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/maxResolution', - v, - ); - }, - }, - ' ': buttonGroup({ - '256px': () => setControls({ max_resolution: 256 }), - '512px': () => setControls({ max_resolution: 512 }), - '1024px': () => setControls({ max_resolution: 1024 }), - '2048px': () => setControls({ max_resolution: 2048 }), - }), - // Enable Crop - crop_enabled: { - label: 'Crop Viewport', - value: crop_enabled, - hint: 'Crop the viewport to the selected box', - onChange: (value) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/crop_enabled', - value, - ); - }, - }, - crop_bg_color: { - label: '| Background Color', - value: crop_bg_color, - render: (get) => get('crop_enabled'), - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/crop_bg_color', - v, - ); - }, - }, - crop_scale: { - label: '| Scale', - value: crop_scale, - min: 0, - max: 10, - step: 0.05, - render: (get) => get('crop_enabled'), - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/crop_scale', - v, - ); - }, - }, - crop_center: { - label: '| Center', - value: crop_center, - min: -10, - max: 10, - step: 0.05, - render: (get) => get('crop_enabled'), - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/crop_center', - v, - ); - }, - }, - // Dynamic NeRF rendering time - ...(display_render_time - ? { - render_time: { - label: 'Render Timestep', - value: render_time, - min: 0, - max: 1, - step: 0.01, - onChange: (v) => { - dispatch_and_send( - websocket, - dispatch, - 'renderingState/render_time', - v, - ); - }, - }, - } - : {}), - }), - [ - outputOptions, - outputChoice, - colormapOptions, - colormapChoice, - max_resolution, - crop_enabled, - target_train_util, - render_time, - display_render_time, - websocket, // need to re-render when websocket changes to use the new websocket - ], + // Leva theming is a bit limited, so we hack at styles here... + return ( + ); - - useEffect(() => { - setControls({ max_resolution }); - setControls({ output_options: outputChoice }); - setControls({ colormap_options: colormapChoice }); - setControls({ crop_enabled }); - setControls({ crop_bg_color }); - setControls({ crop_scale }); - setControls({ crop_center }); - }, [ - setControls, - outputOptions, - outputChoice, - colormapOptions, - colormapChoice, - max_resolution, - target_train_util, - render_time, - crop_enabled, - crop_bg_color, - crop_scale, - crop_center, - display_render_time, - ]); - - return null; } export function RenderControls() { return (
- - + + + +
); } diff --git a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanelSlice.js b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanelSlice.js deleted file mode 100644 index 49f86a065..000000000 --- a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanelSlice.js +++ /dev/null @@ -1,7 +0,0 @@ -// The function below is called a selector and allows us to select a value from -// the state. Selectors can also be defined inline where they're used instead of -// in the slice file. For example: `useSelector((state) => state.counter.value)` -export const selectTrainingState = (state) => - state.shared.rendering.training_state; -export const selectOutputOptions = (state) => state.shared.output_options; -export const selectColormapOptions = (state) => state.shared.colormap_options; diff --git a/nerfstudio/viewer/app/src/modules/LoadPathModal/LoadPathModal.jsx b/nerfstudio/viewer/app/src/modules/LoadPathModal/LoadPathModal.jsx index 5b5d2fced..ad8c83d7e 100644 --- a/nerfstudio/viewer/app/src/modules/LoadPathModal/LoadPathModal.jsx +++ b/nerfstudio/viewer/app/src/modules/LoadPathModal/LoadPathModal.jsx @@ -24,9 +24,7 @@ export default function LoadPathModal(props: LoadPathModalProps) { const [existingPathSelect, setExistingPathSelect] = React.useState(''); // redux store state - const all_camera_paths = useSelector( - (state) => state.renderingState.all_camera_paths, - ); + const all_camera_paths = useSelector((state) => state.all_camera_paths); let camera_paths_arr = []; if (typeof all_camera_paths === 'object' && all_camera_paths !== null) { diff --git a/nerfstudio/viewer/app/src/modules/LogPanel/LogPanel.jsx b/nerfstudio/viewer/app/src/modules/LogPanel/LogPanel.jsx deleted file mode 100644 index 9782a9645..000000000 --- a/nerfstudio/viewer/app/src/modules/LogPanel/LogPanel.jsx +++ /dev/null @@ -1,86 +0,0 @@ -import { useContext, useEffect } from 'react'; -import { useDispatch, useSelector } from 'react-redux'; - -import { WebSocketContext } from '../WebSocket/WebSocket'; - -const msgpack = require('msgpack-lite'); - -export function LogPanel() { - const websocket = useContext(WebSocketContext).socket; - const dispatch = useDispatch(); - const gpu_oom_error_msg = 'GPU out of memory'; - const resolved_msg = 'resolved'; - let local_error = resolved_msg; - // connection status indicators - - const set_max_train_util = () => { - if (websocket.readyState === WebSocket.OPEN) { - dispatch({ - type: 'write', - path: 'renderingState/targetTrainUtil', - data: 0.9, - }); - const cmd = 'write'; - const path = 'renderingState/targetTrainUtil'; - const data = { - type: cmd, - path, - data: 0.9, - }; - const message = msgpack.encode(data); - websocket.send(message); - } - }; - - const set_small_resolution = () => { - if (websocket.readyState === WebSocket.OPEN) { - dispatch({ - type: 'write', - path: 'renderingState/maxResolution', - data: 512, - }); - const cmd = 'write'; - const path = 'renderingState/maxResolution'; - const data = { - type: cmd, - path, - data: 512, - }; - const message = msgpack.encode(data); - websocket.send(message); - } - }; - - const set_log_message = () => { - if (websocket.readyState === WebSocket.OPEN) { - dispatch({ - type: 'write', - path: 'renderingState/log_errors', - data: resolved_msg, - }); - const cmd = 'write'; - const path = 'renderingState/log_errors'; - const data = { - type: cmd, - path, - data: resolved_msg, - }; - const message = msgpack.encode(data); - websocket.send(message); - } - }; - - const check_error = useSelector((state) => { - local_error = state.renderingState.log_errors; - if (local_error.includes(gpu_oom_error_msg)) { - console.log(local_error); - set_log_message(); - set_small_resolution(); - set_max_train_util(); - } - }); - - useEffect(() => {}, [check_error, local_error]); - - return null; -} diff --git a/nerfstudio/viewer/app/src/modules/RenderModal/RenderModal.jsx b/nerfstudio/viewer/app/src/modules/RenderModal/RenderModal.jsx index 8b0a18321..8c6121f68 100644 --- a/nerfstudio/viewer/app/src/modules/RenderModal/RenderModal.jsx +++ b/nerfstudio/viewer/app/src/modules/RenderModal/RenderModal.jsx @@ -16,13 +16,15 @@ export default function RenderModal(props: RenderModalProps) { // redux store state const config_base_dir = useSelector( - (state) => state.renderingState.config_base_dir, + (state) => state.file_path_info.config_base_dir, ); - const export_path = useSelector((state) => state.renderingState.export_path); + const export_path = useSelector( + (state) => state.file_path_info.export_path_name, + ); const data_base_dir = useSelector( - (state) => state.renderingState.data_base_dir, + (state) => state.file_path_info.data_base_dir, ); // react state diff --git a/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx b/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx index e2e1c53e7..598a13298 100644 --- a/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx +++ b/nerfstudio/viewer/app/src/modules/RenderWindow/RenderWindow.jsx @@ -1,12 +1,14 @@ import * as React from 'react'; -import { useSelector } from 'react-redux'; export default function RenderWindow() { - const render_img = useSelector((state) => state.render_img); - return (
- Render window + Render window
); } diff --git a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx index ef5a38888..197f6887f 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx +++ b/nerfstudio/viewer/app/src/modules/Scene/Scene.jsx @@ -1,7 +1,6 @@ /* eslint-disable no-restricted-syntax */ import * as THREE from 'three'; -import { useContext, useEffect } from 'react'; import CameraControls from 'camera-controls'; import { CSS2DRenderer } from 'three/examples/jsm/renderers/CSS2DRenderer'; @@ -11,12 +10,9 @@ import { drawCamera, drawSceneBox } from './drawing'; import { CameraHelper } from '../SidePanel/CameraPanel/CameraHelper'; import SceneNode from '../../SceneNode'; -import { WebSocketContext } from '../WebSocket/WebSocket'; import { subscribe_to_changes } from '../../subscriber'; import { snap_to_camera } from '../SidePanel/SidePanel'; -const msgpack = require('msgpack-lite'); - const SCENE_BOX_NAME = 'Scene Box'; const CAMERAS_NAME = 'Training Cameras'; @@ -41,6 +37,7 @@ export function get_scene_tree() { const sceneTree = new SceneNode(scene, scene_state); + // eslint-disable-next-line react-hooks/rules-of-hooks const dispatch = useDispatch(); const BANNER_HEIGHT = 50; @@ -380,29 +377,3 @@ export function get_scene_tree() { window.addEventListener('mouseup', onMouseUp, false); return sceneTree; } - -// manages setting up the scene and other logic for keeping state in sync with the server -export function SceneTreeWebSocketListener() { - const socket = useContext(WebSocketContext).socket; - const dispatch = useDispatch(); - - useEffect(() => { - const messageHandler = (originalCmd) => { - // set the remote description when the offer is received - const cmd = msgpack.decode(new Uint8Array(originalCmd.data)); - if (cmd.type === 'write') { - // write to the store - dispatch({ - type: 'write', - path: cmd.path, - data: cmd.data, - }); - } - }; - socket.addEventListener('message', messageHandler); - - return () => { - socket.removeEventListener('message', messageHandler); - }; - }, [socket]); // dependency to call this whenever the websocket changes -} diff --git a/nerfstudio/viewer/app/src/modules/Scene/drawing.js b/nerfstudio/viewer/app/src/modules/Scene/drawing.js index fb5ef7af5..9de449a5e 100644 --- a/nerfstudio/viewer/app/src/modules/Scene/drawing.js +++ b/nerfstudio/viewer/app/src/modules/Scene/drawing.js @@ -1,10 +1,9 @@ /* eslint-disable no-restricted-syntax */ // ---- code for drawing with three.js ---- import * as THREE from 'three'; +import { SceneBoxMessage } from '../WebSocket/ViserMessages'; -export function drawSceneBox(sceneBox): THREE.Object3D { - console.assert(sceneBox.type === 'aabb', 'The scene box must be an AABB'); - +export function drawSceneBox(sceneBox: SceneBoxMessage): THREE.Object3D { const box = sceneBox; const w = 1.0; @@ -20,8 +19,8 @@ export function drawSceneBox(sceneBox): THREE.Object3D { points = points.concat([baa, bab, baa, bba, bab, bbb, bba, bbb]); points = points.concat([aaa, baa, aab, bab, aba, bba, abb, bbb]); - const maxPoint = new THREE.Vector3(...box.max_point); - const minPoint = new THREE.Vector3(...box.min_point); + const maxPoint = new THREE.Vector3(...box.max); + const minPoint = new THREE.Vector3(...box.min); const lengths = maxPoint.clone(); lengths.sub(minPoint); diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx index dce2aad0b..8a727be27 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPanel.jsx @@ -45,14 +45,15 @@ import Tooltip from '@mui/material/Tooltip'; import VideoCameraBackIcon from '@mui/icons-material/VideoCameraBack'; import { CameraHelper } from './CameraHelper'; import { get_curve_object_from_cameras, get_transform_matrix } from './curve'; -import { WebSocketContext } from '../../WebSocket/WebSocket'; +import { + ViserWebSocketContext, + sendWebsocketMessage, +} from '../../WebSocket/ViserWebSocket'; import RenderModal from '../../RenderModal'; import LoadPathModal from '../../LoadPathModal'; import CameraPropPanel from './CameraPropPanel'; import LevaTheme from '../../../themes/leva_theme.json'; -const msgpack = require('msgpack-lite'); - const FOV_LABELS = { FOV: '°', MM: 'mm', @@ -64,101 +65,6 @@ function set_camera_position(camera, matrix) { mat.decompose(camera.position, camera.quaternion, camera.scale); } -function RenderTimeSelector(props) { - const disabled = props.disabled; - const isGlobal = props.isGlobal; - const camera = props.camera; - const dispatch = props.dispatch; - const globalRenderTime = props.globalRenderTime; - const setGlobalRenderTime = props.setGlobalRenderTime; - const applyAll = props.applyAll; - const changeMain = props.changeMain; - const setAllCameraRenderTime = props.setAllCameraRenderTime; - - const getRenderTimeLabel = () => { - if (!isGlobal) { - return camera.renderTime; - } - camera.renderTime = globalRenderTime; - return globalRenderTime; - }; - - const [UIRenderTime, setUIRenderTime] = React.useState( - isGlobal ? globalRenderTime : getRenderTimeLabel(), - ); - - const [valid, setValid] = React.useState(true); - - useEffect( - () => setUIRenderTime(getRenderTimeLabel()), - [camera, globalRenderTime], - ); - - const setRndrTime = (val) => { - if (!isGlobal) { - camera.renderTime = val; - } else { - camera.renderTime = val; - setGlobalRenderTime(val); - } - - if (applyAll) { - setAllCameraRenderTime(val); - } - - if (changeMain) { - dispatch({ - type: 'write', - path: 'renderingState/render_time', - data: camera.renderTime, - }); - } - }; - - const handleValidation = (e) => { - const valueFloat = parseFloat(e.target.value); - let valueStr = String(valueFloat); - if (e.target.value >= 0 && e.target.value <= 1) { - setValid(true); - if (valueFloat === 1.0) { - valueStr = '1.0'; - } - if (valueFloat === 0.0) { - valueStr = '0.0'; - } - setUIRenderTime(parseFloat(valueStr)); - setRndrTime(parseFloat(valueStr)); - } else { - setValid(false); - } - }; - - return ( - setUIRenderTime(e.target.value)} - onBlur={(e) => handleValidation(e)} - disabled={disabled} - sx={{ - input: { - WebkitTextFillColor: `${disabled ? '#24B6FF' : '#EBEBEB'} !important`, - color: `${disabled ? '#24B6FF' : '#EBEBEB'} !important`, - }, - }} - value={UIRenderTime} - error={!valid} - helperText={!valid ? 'RenderTime should be between 0.0 and 1.0' : ''} - variant="standard" - /> - ); -} - function FovSelector(props) { const fovLabel = props.fovLabel; const setFovLabel = props.setFovLabel; @@ -197,6 +103,7 @@ function FovSelector(props) { useEffect( () => setUIFieldOfView(getFovLabel()), + // eslint-disable-next-line react-hooks/exhaustive-deps [camera, fovLabel, globalFov], ); @@ -510,16 +417,7 @@ function CameraList(props) { changeMain={false} /> )} - {isAnimated('RenderTime') && ( - - )} - {!isAnimated('FOV') && !isAnimated('RenderTime') && ( + {!isAnimated('FOV') && (

Animated camera properties will show up here!

@@ -552,11 +450,12 @@ export default function CameraPanel(props) { ]); // redux store state - const export_path = useSelector((state) => state.renderingState.export_path); + const export_path = useSelector( + (state) => state.file_path_info.export_path_name, + ); - const websocket = useContext(WebSocketContext).socket; + const viser_websocket = useContext(ViserWebSocketContext); const DEFAULT_FOV = 50; - const DEFAULT_RENDER_TIME = 0.0; // react state const [cameras, setCameras] = React.useState([]); @@ -572,8 +471,6 @@ export default function CameraPanel(props) { const [load_path_modal_open, setLoadPathModalOpen] = React.useState(false); const [animate, setAnimate] = React.useState(new Set()); const [globalFov, setGlobalFov] = React.useState(DEFAULT_FOV); - const [globalRenderTime, setGlobalRenderTime] = - React.useState(DEFAULT_RENDER_TIME); // leva store const cameraPropsStore = useCreateStore(); @@ -588,6 +485,7 @@ export default function CameraPanel(props) { (value) => setMouseInScene(value), 'mouse_in_scene', ); + // eslint-disable-next-line react-hooks/exhaustive-deps }, []); const dispatch = useDispatch(); @@ -608,17 +506,6 @@ export default function CameraPanel(props) { const crop_center = useSelector((state) => state.renderingState.crop_center); const crop_scale = useSelector((state) => state.renderingState.crop_scale); - const [display_render_time, set_display_render_time] = React.useState(false); - - const receive_temporal_dist = (e) => { - const msg = msgpack.decode(new Uint8Array(e.data)); - if (msg.path === '/model/has_temporal_distortion') { - set_display_render_time(msg.data === 'true'); - websocket.removeEventListener('message', receive_temporal_dist); - } - }; - websocket.addEventListener('message', receive_temporal_dist); - const setRenderHeight = (value) => { dispatch({ type: 'write', @@ -650,46 +537,6 @@ export default function CameraPanel(props) { }); }; - const setCropEnabled = (value) => { - dispatch({ - type: 'write', - path: 'renderingState/crop_enabled', - data: value, - }); - }; - - const serCropBgColor = (value) => { - dispatch({ - type: 'write', - path: 'renderingState/crop_bg_color', - data: value, - }); - }; - - const setCropCenter = (value) => { - dispatch({ - type: 'write', - path: 'renderingState/crop_center', - data: value, - }); - }; - - const setCropScale = (value) => { - dispatch({ - type: 'write', - path: 'renderingState/crop_scale', - data: value, - }); - }; - - const setRenderTime = (value) => { - dispatch({ - type: 'write', - path: 'renderingState/render_time', - data: parseFloat(value), - }); - }; - // ui state const [fovLabel, setFovLabel] = React.useState(FOV_LABELS.FOV); @@ -706,7 +553,6 @@ export default function CameraPanel(props) { if (new_camera_list.length >= 1) { set_camera_position(camera_render, new_camera_list[0].matrix); setFieldOfView(new_camera_list[0].fov); - setRenderTime(new_camera_list[0].renderTime); set_slider_value(slider_min); } }; @@ -715,7 +561,7 @@ export default function CameraPanel(props) { const camera_main_copy = camera_main.clone(); camera_main_copy.aspect = 1.0; camera_main_copy.fov = globalFov; - camera_main_copy.renderTime = globalRenderTime; + camera_main_copy.renderTime = 0.0; const new_camera_properties = new Map(); camera_main_copy.properties = new_camera_properties; new_camera_properties.set('FOV', globalFov); @@ -845,6 +691,7 @@ export default function CameraPanel(props) { camera_helper, ); } + // eslint-disable-next-line react-hooks/exhaustive-deps }, [cameras, cameraProperties, render_width, render_height]); // update the camera curve @@ -962,6 +809,7 @@ export default function CameraPanel(props) { setFieldOfView(fov); setGlobalFov(fov); } + // eslint-disable-next-line react-hooks/exhaustive-deps }, [slider_value, render_height, render_width]); // call this function whenever slider state changes @@ -973,6 +821,7 @@ export default function CameraPanel(props) { return () => clearInterval(interval); } return () => {}; + // eslint-disable-next-line react-hooks/exhaustive-deps }, [is_playing]); // make sure to pause if the slider reaches the end @@ -998,21 +847,11 @@ export default function CameraPanel(props) { const mat = get_transform_matrix(position, lookat, up); - if (display_render_time) { - const renderTime = curve_object.curve_render_times.getPoint(pt).z; - camera_path.push({ - camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix - fov, - aspect: camera_render.aspect, - render_time: Math.max(Math.min(renderTime, 1.0), 0.0), // clamp time values to [0, 1] - }); - } else { - camera_path.push({ - camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix - fov, - aspect: camera_render.aspect, - }); - } + camera_path.push({ + camera_to_world: mat.transpose().elements, // convert from col-major to row-major matrix + fov, + aspect: camera_render.aspect, + }); } const keyframes = []; @@ -1029,7 +868,11 @@ export default function CameraPanel(props) { let crop = null; if (crop_enabled) { crop = { - crop_bg_color, + crop_bg_color: { + r: crop_bg_color[0], + g: crop_bg_color[1], + b: crop_bg_color[2], + }, crop_center, crop_scale, }; @@ -1056,6 +899,8 @@ export default function CameraPanel(props) { // inspired by: // https://stackoverflow.com/questions/55613438/reactwrite-to-json-file-or-export-download-no-server + sendWebsocketMessage(viser_websocket, { type: 'SaveCheckpointMessage' }); + const camera_path_object = get_camera_path(); console.log(camera_render.toJSON()); @@ -1115,10 +960,14 @@ export default function CameraPanel(props) { reset_slider_render_on_add(new_camera_list); if ('crop' in camera_path_object && camera_path_object.crop !== null) { - setCropEnabled(true); - serCropBgColor(camera_path_object.crop.crop_bg_color); - setCropCenter(camera_path_object.crop.crop_center); - setCropScale(camera_path_object.crop.crop_scale); + const bg_color = camera_path_object.crop.crop_bg_color; + sendWebsocketMessage(viser_websocket, { + type: 'crop_params', + crop_enabled: true, + crop_bg_color: [bg_color.r, bg_color.g, bg_color.b], + crop_center: camera_path_object.crop.crop_center, + crop_scale: camera_path_object.crop.crop_scale, + }); } }; @@ -1138,33 +987,17 @@ export default function CameraPanel(props) { setRenderModalOpen(true); const camera_path_object = get_camera_path(); - const camera_path_payload = { + + sendWebsocketMessage(viser_websocket, { + type: 'CameraPathPayloadMessage', camera_path_filename: export_path, camera_path: camera_path_object, - }; - - // send a command of the websocket to save the trajectory somewhere! - if (websocket.readyState === WebSocket.OPEN) { - const data = { - type: 'write', - path: 'camera_path_payload', - data: camera_path_payload, - }; - const message = msgpack.encode(data); - websocket.send(message); - } + }); + sendWebsocketMessage(viser_websocket, { type: 'SaveCheckpointMessage' }); }; const open_load_path_modal = () => { - if (websocket.readyState === WebSocket.OPEN) { - const data = { - type: 'write', - path: 'populate_paths_payload', - data: true, - }; - const message = msgpack.encode(data); - websocket.send(message); - } + sendWebsocketMessage(viser_websocket, { type: 'CameraPathOptionsRequest' }); setLoadPathModalOpen(true); }; @@ -1193,12 +1026,6 @@ export default function CameraPanel(props) { } }; - const setAllCameraRenderTime = (val) => { - for (let i = 0; i < cameras.length; i += 1) { - cameras[i].renderTime = val; - } - }; - return (
@@ -1263,51 +1090,6 @@ export default function CameraPanel(props) { />
- {display_render_time && ( -
- - - - -
- )} {camera_type !== 'equirectangular' && (
diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPropPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPropPanel.jsx index 84a5cfaed..8bafc3b34 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPropPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/CameraPanel/CameraPropPanel.jsx @@ -21,12 +21,14 @@ export default function CameraPropPanel(props) { ); const camera_type = useSelector((state) => state.renderingState.camera_type); - const export_path = useSelector((state) => state.renderingState.export_path); + const export_path = useSelector( + (state) => state.file_path_info.export_path_name, + ); const setExportPath = (value) => { dispatch({ type: 'write', - path: 'renderingState/export_path', + path: 'file_path_info/export_path_name', data: value, }); }; @@ -59,14 +61,12 @@ export default function CameraPropPanel(props) { value: export_path, onChange: (v) => { const valid_filename_reg = /^([a-z]|[A-Z]|[0-9]|-|_)+$/g; - if(!valid_filename_reg.test(v)){ - alert("Please only use letters, numbers, and hyphens"); - } - else { + if (!valid_filename_reg.test(v)) { + alert('Please only use letters, numbers, and hyphens'); + } else { setExportPath(v); } }, - }, camera_resolution: { label: 'Resolution', @@ -109,7 +109,7 @@ export default function CameraPropPanel(props) { { store }, ); - setControls({path: export_path}); + setControls({ path: export_path }); setControls({ video_fps: fps }); setControls({ video_duration: seconds }); setControls({ diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/ExportPanel/MeshSubPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/ExportPanel/MeshSubPanel.jsx index f345652e8..c322ce971 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/ExportPanel/MeshSubPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/ExportPanel/MeshSubPanel.jsx @@ -33,7 +33,7 @@ export default function MeshSubPanel(props) { // redux store state const config_base_dir = useSelector( - (state) => state.renderingState.config_base_dir, + (state) => state.file_path_info.config_base_dir, ); const clippingEnabled = useSelector( (state) => state.renderingState.clipping_enabled, diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/ExportPanel/PointcloudSubPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/ExportPanel/PointcloudSubPanel.jsx index 480f3694c..9a8f7177e 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/ExportPanel/PointcloudSubPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/ExportPanel/PointcloudSubPanel.jsx @@ -15,7 +15,7 @@ export default function PointcloudSubPanel(props) { // redux store state const config_base_dir = useSelector( - (state) => state.renderingState.config_base_dir, + (state) => state.file_path_info.config_base_dir, ); const clippingEnabled = useSelector( (state) => state.renderingState.clipping_enabled, diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/ScenePanel/ScenePanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/ScenePanel/ScenePanel.jsx index 233520125..254d75008 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/ScenePanel/ScenePanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/ScenePanel/ScenePanel.jsx @@ -64,11 +64,13 @@ function MenuItems(props: ListItemProps) { return null; } + // eslint-disable-next-line react-hooks/rules-of-hooks const [open, setOpen] = React.useState(true); const toggleOpen = () => { setOpen(!open); }; + // eslint-disable-next-line react-hooks/rules-of-hooks const [visible, setVisible] = React.useState(groupVisible); if (terminal) { scene_node.object.visible = visible; @@ -77,16 +79,15 @@ function MenuItems(props: ListItemProps) { const toggleVisible = (e) => { e.stopPropagation(); setVisible(!visible); - scene_node.object.traverse( - (obj) => { - if (obj.name === 'CAMERA_LABEL') { - // eslint-disable-next-line no-param-reassign - obj.visible = !visible; - } + scene_node.object.traverse((obj) => { + if (obj.name === 'CAMERA_LABEL') { + // eslint-disable-next-line no-param-reassign + obj.visible = !visible; } - ); + }); }; + // eslint-disable-next-line react-hooks/rules-of-hooks React.useEffect(() => { setVisible(groupVisible); }, [groupVisible]); diff --git a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx index b80446664..a330f0311 100644 --- a/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/SidePanel/StatusPanel/StatusPanel.jsx @@ -6,10 +6,10 @@ import PauseIcon from '@mui/icons-material/Pause'; import PlayArrowIcon from '@mui/icons-material/PlayArrow'; import ViewInArIcon from '@mui/icons-material/ViewInAr'; import ImageOutlinedIcon from '@mui/icons-material/ImageOutlined'; - -import { WebSocketContext } from '../../WebSocket/WebSocket'; - -const msgpack = require('msgpack-lite'); +import { + ViserWebSocketContext, + sendWebsocketMessage, +} from '../../WebSocket/ViserWebSocket'; interface StatusPanelProps { sceneTree: object; @@ -17,17 +17,15 @@ interface StatusPanelProps { export default function StatusPanel(props: StatusPanelProps) { const dispatch = useDispatch(); - const websocket = React.useContext(WebSocketContext).socket; + const viser_websocket = React.useContext(ViserWebSocketContext); const isTraining = useSelector((state) => state.renderingState.isTraining); const sceneTree = props.sceneTree; const isWebsocketConnected = useSelector( (state) => state.websocketState.isConnected, ); + const step = useSelector((state) => state.renderingState.step); const eval_res = useSelector((state) => state.renderingState.eval_res); - const vis_train_ratio = useSelector( - (state) => state.renderingState.vis_train_ratio, - ); const camera_choice = useSelector( (state) => state.renderingState.camera_choice, ); @@ -51,6 +49,7 @@ export default function StatusPanel(props: StatusPanelProps) { obj.visible = is_scene_visible && camera_choice === 'Main Camera'; } }); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [camera_choice, is_scene_visible]); const handlePlayChange = () => { @@ -59,16 +58,10 @@ export default function StatusPanel(props: StatusPanelProps) { path: 'renderingState/isTraining', data: !isTraining, }); - // write to server - const cmd = 'write'; - const path = 'renderingState/isTraining'; - const data = { - type: cmd, - path, - data: !isTraining, - }; - const message = msgpack.encode(data); - websocket.send(message); + sendWebsocketMessage(viser_websocket, { + type: 'IsTrainingMessage', + is_training: !isTraining, + }); }; const is_training_text = isTraining ? 'Pause Training' : 'Resume Training'; const training_icon = isTraining ? : ; @@ -129,10 +122,10 @@ export default function StatusPanel(props: StatusPanelProps) {
- Resolution: {eval_res} + Iteration: {step}
- Time Allocation: {vis_train_ratio} + Resolution: {eval_res}