mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-17 21:33:56 +08:00
Viewer backend refactor (#1702)
* Copy in `viser` communication infrastructure * Update background image * Add build directory to fix viser HTTP server May want to remove these later * Copy in `viser` communication infrastructure * Update background image * Pass dataset vis using viser * initial impl of ViewerParameter * idk what i did but brent and matt know * Suppress eslint typescript warnings * implement ViewerElement base class, then ViewerButton, ViewerSlider, etc * client -> server message setup * multi folder stuff * Stream controls over viser * import * add types to viewer_param.py * Move another message to viser * Update folder color * Use non throttled messages * multiple folders works * Cleanup, more porting * starting impl of control panel in viser * More porting * scenebox to viser * Cleanup * Fix render updates * More cleanup, docs, typing * start control_panel.py * set_hidden logic but not merged in * control_panel replicates behavior of existing control panel (visually), except for setting the list of output options. not yet integrated with viewer_utils.py for logic flow. also implemented set_disabled and set_hidden in javscript/ViewerElement side * Remove some original leva components * add some train util logic, attempt set_value, doesn't work on javascript side fully yet (value updates but not slider visually * bye zmq * missing call to decoder * 'gui_set' works properly, control panel now replicates behavior of original * remove print * Fix status viewer message * More cleanup * support set_options in dropdown * hiding buttons works correctly * fix set_options, switch from cur_value to get_value() * Fix slowdown * output options * colormap logic converted * add handle to cb_hook * fix some lambda bugs * cleanup * Save checkpoint logic readded * Cleanup and minor fixes * snappier ui * Refactoring * Fix state update bugs * Bug fixes * Write less to redux * added comments to reducer code * Render state machine (#1727) * state machine works, but train step throttling not implemented * more step logic, cleanup * remove random try catcH * mutex logic for render * Update lock logic * types and docs * added some state machine logic to help rerenders * merge --------- Co-authored-by: Justin <justin.g.kerr@gmail.com> * Refactor and cleanup * More cleanup and docs * docstring to gui_utils * docstrings, rename viewer_param.py to viewer_elements.py * Fix viewer, remove bridge stuff * Render on connect * Fix lint errors * More eslint fixes * render_freq logic * better rerender logic for crop * More cleanup * Purge * Cleanup * Lint fixes * Fix cache * better parsing logic * Fix infinite test * More cleanup * typo * Update webdocs * Update docs * rerender on custom hooks * add more docs * Use shared utilities in `viser.infra` * rework callback to use elements instead of handles * Fix ESlint errors * change naming of handle to element, add setter for value * cleanup --------- Co-authored-by: Matt Tancik <mattt4001@gmail.com> Co-authored-by: Justin <justin.g.kerr@gmail.com> Co-authored-by: Ethan Weber <eweb0124@gmail.com> Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
This commit is contained in:
68 files changed
+3700
-2584
No files matched your search
+1
-1
@@ -27,7 +27,7 @@ class ComputePSNR:
|
||||
|
||||
def main(self) -> None:
|
||||
"""Main function."""
|
||||
config, pipeline, checkpoint_path = eval_setup(self.load_config)
|
||||
config, pipeline, checkpoint_path, _ = eval_setup(self.load_config)
|
||||
assert self.output_path.suffix == ".json"
|
||||
metrics_dict = pipeline.get_average_eval_image_metrics()
|
||||
self.output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
+5
-5
@@ -77,7 +77,7 @@ class ExportPointCloud(Exporter):
|
||||
if not self.output_dir.exists():
|
||||
self.output_dir.mkdir(parents=True)
|
||||
|
||||
_, pipeline, _ = eval_setup(self.load_config)
|
||||
_, pipeline, _, _ = eval_setup(self.load_config)
|
||||
|
||||
# Increase the batchsize to speed up the evaluation.
|
||||
pipeline.datamanager.train_pixel_sampler.num_rays_per_batch = self.num_rays_per_batch
|
||||
@@ -147,7 +147,7 @@ class ExportTSDFMesh(Exporter):
|
||||
if not self.output_dir.exists():
|
||||
self.output_dir.mkdir(parents=True)
|
||||
|
||||
_, pipeline, _ = eval_setup(self.load_config)
|
||||
_, pipeline, _, _ = eval_setup(self.load_config)
|
||||
|
||||
tsdf_utils.export_tsdf_mesh(
|
||||
pipeline,
|
||||
@@ -253,7 +253,7 @@ class ExportPoissonMesh(Exporter):
|
||||
if not self.output_dir.exists():
|
||||
self.output_dir.mkdir(parents=True)
|
||||
|
||||
_, pipeline, _ = eval_setup(self.load_config)
|
||||
_, pipeline, _, _ = eval_setup(self.load_config)
|
||||
self.validate_pipeline(pipeline)
|
||||
|
||||
# Increase the batchsize to speed up the evaluation.
|
||||
@@ -342,7 +342,7 @@ class ExportMarchingCubesMesh(Exporter):
|
||||
if not self.output_dir.exists():
|
||||
self.output_dir.mkdir(parents=True)
|
||||
|
||||
_, pipeline, _ = eval_setup(self.load_config)
|
||||
_, pipeline, _, _ = eval_setup(self.load_config)
|
||||
|
||||
# TODO: Make this work with Density Field
|
||||
assert hasattr(pipeline.model.config, "sdf_field"), "Model must have an SDF field."
|
||||
@@ -391,7 +391,7 @@ class ExportCameraPoses(Exporter):
|
||||
if not self.output_dir.exists():
|
||||
self.output_dir.mkdir(parents=True)
|
||||
|
||||
_, pipeline, _ = eval_setup(self.load_config)
|
||||
_, pipeline, _, _ = eval_setup(self.load_config)
|
||||
assert isinstance(pipeline, VanillaPipeline)
|
||||
train_frames, eval_frames = collect_camera_poses(pipeline)
|
||||
|
||||
|
||||
+1
-1
@@ -283,7 +283,7 @@ class RenderTrajectory:
|
||||
|
||||
def main(self) -> None:
|
||||
"""Main function."""
|
||||
_, pipeline, _ = eval_setup(
|
||||
_, pipeline, _, _ = eval_setup(
|
||||
self.load_config,
|
||||
eval_num_rays_per_chunk=self.eval_num_rays_per_chunk,
|
||||
test_mode="test" if self.traj in ["spiral", "interpolate"] else "inference",
|
||||
|
||||
+1
-1
@@ -51,7 +51,7 @@ class TextureMesh:
|
||||
mesh = get_mesh_from_filename(str(self.input_mesh_filename), target_num_faces=self.target_num_faces)
|
||||
|
||||
# load the Pipeline
|
||||
_, pipeline, _ = eval_setup(self.load_config, test_mode="inference")
|
||||
_, pipeline, _, _ = eval_setup(self.load_config, test_mode="inference")
|
||||
|
||||
# texture the mesh with NeRF and export to a mesh.obj file
|
||||
# and a material and texture file
|
||||
|
||||
@@ -16,8 +16,7 @@ from nerfstudio.engine.trainer import TrainerConfig
|
||||
from nerfstudio.pipelines.base_pipeline import Pipeline
|
||||
from nerfstudio.utils import writer
|
||||
from nerfstudio.utils.eval_utils import eval_setup
|
||||
from nerfstudio.utils.writer import EventName, TimeWriter
|
||||
from nerfstudio.viewer.server import viewer_utils
|
||||
from nerfstudio.viewer.server.viewer_state import ViewerState
|
||||
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
@@ -27,7 +26,6 @@ class ViewerConfigWithoutNumRays(ViewerConfig):
|
||||
"""Configuration for viewer instantiation"""
|
||||
|
||||
num_rays_per_chunk: tyro.conf.Suppress[int] = -1
|
||||
start_train: tyro.conf.Suppress[bool] = False
|
||||
|
||||
def as_viewer_config(self):
|
||||
"""Converts the instance to ViewerConfig"""
|
||||
@@ -45,7 +43,7 @@ class RunViewer:
|
||||
|
||||
def main(self) -> None:
|
||||
"""Main function."""
|
||||
config, pipeline, _ = eval_setup(
|
||||
config, pipeline, _, step = eval_setup(
|
||||
self.load_config,
|
||||
eval_num_rays_per_chunk=None,
|
||||
test_mode="test",
|
||||
@@ -56,45 +54,7 @@ class RunViewer:
|
||||
config.viewer = self.viewer.as_viewer_config()
|
||||
config.viewer.num_rays_per_chunk = num_rays_per_chunk
|
||||
|
||||
self._start_viewer(config, pipeline)
|
||||
|
||||
def _start_viewer(self, config: TrainerConfig, pipeline: Pipeline):
|
||||
base_dir = config.get_base_dir()
|
||||
viewer_log_path = base_dir / config.viewer.relative_log_filename
|
||||
viewer_state, banner_messages = viewer_utils.setup_viewer(
|
||||
config.viewer, log_filename=viewer_log_path, datapath=pipeline.datamanager.get_datapath()
|
||||
)
|
||||
|
||||
# We don't need logging, but writer.GLOBAL_BUFFER needs to be populated
|
||||
config.logging.local_writer.enable = False
|
||||
writer.setup_local_writer(config.logging, max_iter=config.max_num_iterations, banner_messages=banner_messages)
|
||||
|
||||
assert viewer_state and pipeline.datamanager.train_dataset
|
||||
viewer_state.init_scene(
|
||||
dataset=pipeline.datamanager.train_dataset,
|
||||
start_train=False,
|
||||
)
|
||||
while True:
|
||||
viewer_state.vis["renderingState/isTraining"].write(False)
|
||||
self._update_viewer_state(viewer_state, pipeline)
|
||||
|
||||
def _update_viewer_state(self, viewer_state: viewer_utils.ViewerState, pipeline: Pipeline):
|
||||
"""Updates the viewer state by rendering out scene with current pipeline
|
||||
Returns the time taken to render scene.
|
||||
|
||||
"""
|
||||
# NOTE: step must be > 0 otherwise the rendering would not happen
|
||||
step = 1
|
||||
num_rays_per_batch = pipeline.datamanager.get_train_rays_per_batch()
|
||||
with TimeWriter(writer, EventName.ITER_VIS_TIME) as _:
|
||||
try:
|
||||
viewer_state.update_scene(self, step, pipeline.model, num_rays_per_batch)
|
||||
except RuntimeError:
|
||||
time.sleep(0.03) # sleep to allow buffer to reset
|
||||
assert viewer_state.vis is not None
|
||||
viewer_state.vis["renderingState/log_errors"].write(
|
||||
"Error: GPU out of memory. Reduce resolution to prevent viewer from crashing."
|
||||
)
|
||||
_start_viewer(config, pipeline, step)
|
||||
|
||||
def save_checkpoint(self, *args, **kwargs):
|
||||
"""
|
||||
@@ -102,6 +62,39 @@ class RunViewer:
|
||||
"""
|
||||
|
||||
|
||||
def _start_viewer(config: TrainerConfig, pipeline: Pipeline, step: int):
|
||||
"""Starts the viewer
|
||||
|
||||
Args:
|
||||
config: Configuration of pipeline to load
|
||||
pipeline: Pipeline instance of which to load weights
|
||||
step: Step at which the pipeline was saved
|
||||
"""
|
||||
base_dir = config.get_base_dir()
|
||||
viewer_log_path = base_dir / config.viewer.relative_log_filename
|
||||
viewer_state = ViewerState(
|
||||
config.viewer,
|
||||
log_filename=viewer_log_path,
|
||||
datapath=pipeline.datamanager.get_datapath(),
|
||||
pipeline=pipeline,
|
||||
)
|
||||
banner_messages = [f"Viewer at: {viewer_state.viewer_url}"]
|
||||
|
||||
# We don't need logging, but writer.GLOBAL_BUFFER needs to be populated
|
||||
config.logging.local_writer.enable = False
|
||||
writer.setup_local_writer(config.logging, max_iter=config.max_num_iterations, banner_messages=banner_messages)
|
||||
|
||||
assert viewer_state and pipeline.datamanager.train_dataset
|
||||
viewer_state.init_scene(
|
||||
dataset=pipeline.datamanager.train_dataset,
|
||||
start_train=False,
|
||||
)
|
||||
viewer_state.viser_server.set_is_training(False)
|
||||
viewer_state.update_scene(step=step)
|
||||
while True:
|
||||
time.sleep(0.01)
|
||||
|
||||
|
||||
def entrypoint():
|
||||
"""Entrypoint for use with pyproject scripts."""
|
||||
tyro.extras.set_accent_color("bright_yellow")
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Generate viser message definitions for TypeScript, by parsing Python dataclasses."""
|
||||
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
from viser.infra import generate_typescript_interfaces
|
||||
|
||||
from nerfstudio.viewer.viser import NerfstudioMessage
|
||||
|
||||
|
||||
def entrypoint() -> None:
|
||||
"""Generate viser message definitions for TypeScript, by parsing Python dataclasses."""
|
||||
|
||||
# Generate typescript source.
|
||||
defs = generate_typescript_interfaces(NerfstudioMessage)
|
||||
|
||||
# Write to file.
|
||||
# Three parents from nerfstudio/scripts/viewer/sync_viser_message_defs.py:
|
||||
# - nerfstudio/scripts/viewer/
|
||||
# - nerfstudio/scripts/
|
||||
# - nerfstudio/
|
||||
target_path = pathlib.Path(__file__).absolute().parent.parent.parent / pathlib.Path(
|
||||
"nerfstudio/viewer/app/src/modules/WebSocket/ViserMessages.tsx"
|
||||
)
|
||||
assert target_path.exists()
|
||||
target_path.write_text(defs, encoding="utf_8")
|
||||
print(f"Wrote to {target_path}")
|
||||
|
||||
# Run prettier.
|
||||
# TODO: if this is not installed maybe we should print some error with installation
|
||||
# instructions?
|
||||
subprocess.run(args=["prettier", "-w", str(target_path)], check=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
entrypoint()
|
||||
@@ -1,45 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
view_dataset.py
|
||||
"""
|
||||
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import tyro
|
||||
from rich.console import Console
|
||||
|
||||
from nerfstudio.configs.base_config import ViewerConfig
|
||||
from nerfstudio.data.datamanagers.base_datamanager import AnnotatedDataParserUnion
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.viewer.server import viewer_utils
|
||||
|
||||
DEFAULT_TIMEOUT = timedelta(minutes=30)
|
||||
CONSOLE = Console(width=120)
|
||||
|
||||
# speedup for when input size to model doesn't change (much)
|
||||
torch.backends.cudnn.benchmark = True # type: ignore
|
||||
|
||||
|
||||
def main(
|
||||
dataparser: AnnotatedDataParserUnion,
|
||||
viewer: ViewerConfig,
|
||||
log_base_dir: Path = Path("/tmp/nerfstudio_viewer_logs"),
|
||||
) -> None:
|
||||
"""Main function."""
|
||||
viewer_state, _ = viewer_utils.setup_viewer(
|
||||
viewer,
|
||||
log_filename=log_base_dir / viewer.relative_log_filename,
|
||||
datapath=dataparser.data,
|
||||
)
|
||||
dataset = InputDataset(dataparser.setup().get_dataparser_outputs(split="train"))
|
||||
viewer_state.init_scene(dataset=dataset, start_train=False)
|
||||
CONSOLE.log("Please refresh and load page at: %s", viewer_state.viewer_url)
|
||||
time.sleep(30) # allowing time to refresh page
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
tyro.extras.set_accent_color("bright_yellow")
|
||||
tyro.cli(main)
|
||||
Reference in new issue
Block a user