Files
nerfstudio/notebooks/visualize_viewer.ipynb
T
Ethan Weberandevonneng bf13ed3cf3 Integrating WebRTC into viewer (#73)
* restructuring for webrtc integration

* some webrtc code but not working

* webrtc connects when muted

* the flag is real!

* updating camera in websocket bridge. moved webrtc to existing websocket connection instead of http request. need to change video track

* rudimentary adding real-time imaging

* added free TURN server so code works everywhere

* slow rendering works during training with the viewer

* fix tensor device and update default model save dir

* forgot to include trainer updates

* removing some code that isn't needed

* quick fix to bins. not the most accurate however

* cleaned up code and added some config logic

* black

* adding license

Co-authored-by: evonneng <evonne_ng@berkeley.edu>
2022-07-05 14:52:49 -07:00

271 lines
7.5 KiB
Plaintext

{
"cells": [
{
"cell_type": "markdown",
"id": "3c00934d-0f14-4c1a-a563-94d627b377e5",
"metadata": {},
"source": [
"## Notebook to visualize the viewer functionality\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8501e01d-f932-4a75-8259-2def67a546d7",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b15dfdc1-5054-40bc-9631-4c7426eecfd9",
"metadata": {},
"outputs": [],
"source": [
"from notebook_imports import *\n",
"\n",
"import time\n",
"import torch\n",
"from torch.utils.data import DataLoader\n",
"\n",
"from pyrad.data.dataloader import TrainDataloader\n",
"from pyrad.data.image_dataset import ImageDataset, PanopticImageDataset\n",
"from pyrad.data.image_sampler import CacheImageSampler\n",
"from pyrad.data.pixel_sampler import PixelSampler\n",
"from pyrad.data.utils import DatasetInputs, get_dataset_inputs_from_dataset_config\n",
"from pyrad.graphs.modules.ray_generator import RayGenerator\n",
"from pyrad.graphs.modules.scene_colliders import SceneBoundsCollider, AABBBoxCollider\n",
"from pyrad.cameras.cameras import get_camera\n",
"from pyrad.cameras.camera_paths import InterpolatedCameraPath\n",
"from pyrad.cameras.rays import RayBundle\n",
"from pyrad.utils.io import get_absolute_path\n",
"from pyrad.utils.plotly import get_line_segments_from_lines\n",
"from pyrad.cameras.cameras import get_camera_model\n",
"from pyrad.utils.misc import get_dict_to_torch, instantiate_from_dict_config\n",
"from pyrad.engine.trainer import Trainer\n",
"from pyrad.utils.config import Config, setup_config\n",
"\n",
"# the new import\n",
"from pyrad.viewer.backend import vis_utils\n",
"\n",
"\n",
"from hydra import compose, initialize\n",
"from omegaconf import open_dict\n",
"import pprint\n",
"from tqdm import tqdm\n",
"import random\n",
"\n",
"import umsgpack\n",
"from pyrad.viewer.backend.utils import get_chunks, get_intrinsics_matrix_and_camera_to_world_h"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "032e4c9e-25be-4cc0-9ac8-a5be723b775a",
"metadata": {},
"outputs": [],
"source": [
"# establish the connection to the tcp backend, to talk to the visualizer\n",
"# zmq_url should match the output of `python run_zmq_server.py`\n",
"vis = vis_utils.get_vis(zmq_url=\"tcp://127.0.0.1:6000\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df82e083-e261-43e0-a1dc-05c6a59d16e1",
"metadata": {},
"outputs": [],
"source": [
"vis.delete()\n",
"\n",
"# draws a red box in the scene\n",
"vis_utils.show_box_test(vis)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7fe7478d",
"metadata": {},
"outputs": [],
"source": [
"with initialize(version_base=None, config_path=\"../configs\"):\n",
" config = compose(\n",
" config_name=\"graph_default.yaml\",\n",
" overrides=[\n",
" \"trainer.resume_train.load_dir=/shared/ethanweber/pyrad/outputs/blender_lego/vanilla_nerf/2022-07-04_221907/checkpoints\",\n",
" \"trainer.resume_train.load_step=4000\",\n",
" ],\n",
" )\n",
" config = setup_config(config)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5d231e36-dd38-489b-a3a3-860cdf49e551",
"metadata": {},
"outputs": [],
"source": [
"trainer = Trainer(config, local_rank=0, world_size=1)\n",
"trainer.setup()\n",
"trainer.draw_scene_in_viewer()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1aba9ecb-0022-4896-833d-577abe532d23",
"metadata": {},
"outputs": [],
"source": [
"aabb = trainer.dataset_inputs_train.scene_bounds.aabb\n",
"vis_utils.draw_aabb(trainer.vis, aabb, name=\"dataset_inputs_train/scene_bounds/aabb\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4b23fc89-bfc3-40c4-abba-fad1317aeb21",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5a829d31-4fc6-4908-963d-7326d3aa52da",
"metadata": {},
"outputs": [],
"source": [
"outputs = trainer.render_image_in_viewer()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "84bd64de-2e0c-458e-a4b7-1084f1cca1c5",
"metadata": {},
"outputs": [],
"source": [
"trainer.vis.delete()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6ed9cdfa-6359-4a70-95c7-033e853e9b50",
"metadata": {},
"outputs": [],
"source": [
"while True:\n",
" outputs = trainer.render_image_in_viewer()\n",
" image = outputs[\"rgb_fine\"].cpu().numpy().astype(\"float64\") * 255\n",
" trainer.vis[\"/Cameras/Main Camera\"].set_image(image)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b3bcd221-05a8-494e-b817-3e2de7f5a0df",
"metadata": {},
"outputs": [],
"source": [
"image.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "642b0a25-0e88-4eac-b661-a57a74fb10fc",
"metadata": {},
"outputs": [],
"source": [
"dataset_inputs = get_dataset_inputs_from_dataset_config(**config.data.dataset_inputs_train, split=\"train\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "938f090b-3b28-4fd6-82dc-d4b5c5917134",
"metadata": {},
"outputs": [],
"source": [
"# ImageDataset\n",
"image_dataset_train = instantiate_from_dict_config(config.data.image_dataset_train, **dataset_inputs.as_dict())"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "fd04703a-aa50-45e4-95fd-4d881ff18d61",
"metadata": {},
"outputs": [],
"source": [
"# move a camera around\n",
"num_cameras = len(dataset_inputs.intrinsics)\n",
"intrinsics = dataset_inputs.intrinsics\n",
"camera_to_world = dataset_inputs.camera_to_world\n",
"idx0, idx1 = random.sample(range(num_cameras), k=2)\n",
"camera_a = get_camera(intrinsics[idx0], camera_to_world[idx0], None)\n",
"camera_b = get_camera(intrinsics[idx1], camera_to_world[idx1], None)\n",
"\n",
"num_steps = 100\n",
"fps = 30\n",
"estimated_time = num_steps / fps\n",
"print(\"estimated_time:\", estimated_time)\n",
"\n",
"camera_path = InterpolatedCameraPath(camera_a, camera_b)\n",
"cameras = camera_path.get_path(steps=num_steps)\n",
"\n",
"start = time.time()\n",
"for camera in cameras:\n",
" vis_utils.set_camera(vis, camera)\n",
" time.sleep(1 / fps)\n",
"time_elapsed = time.time() - start\n",
"print(\"time_elapsed:\", time_elapsed)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f1cd7183-c514-4f87-a8cc-e14228228e24",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"interpreter": {
"hash": "4a749acc7d255f078aee52e0584cc77b3cb5aaed1b3a7407ec4262c1bf6cb526"
},
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}