nerfstudio/notebooks/visualize_viewer.ipynb

Notebook to visualize the viewer functionality

In [1]:

from notebook_imports import *

import time
import torch
from torch.utils.data import DataLoader

from pyrad.data.dataloader import TrainDataloader
from pyrad.data.image_dataset import ImageDataset, PanopticImageDataset
from pyrad.data.image_sampler import CacheImageSampler
from pyrad.data.pixel_sampler import PixelSampler
from pyrad.data.utils import DatasetInputs, get_dataset_inputs_from_dataset_config
from pyrad.graphs.modules.ray_generator import RayGenerator
from pyrad.graphs.modules.scene_colliders import SceneBoundsCollider, AABBBoxCollider
from pyrad.cameras.cameras import get_camera
from pyrad.cameras.camera_paths import InterpolatedCameraPath
from pyrad.cameras.rays import RayBundle
from pyrad.utils.io import get_absolute_path
from pyrad.utils.plotly import get_line_segments_from_lines
from pyrad.cameras.cameras import get_camera_model
from pyrad.utils.misc import get_dict_to_torch, instantiate_from_dict_config

# the new import
from pyrad.viewer.backend import vis_utils


from hydra import compose, initialize
from omegaconf import open_dict
import pprint
from tqdm import tqdm
import random

In [2]:

# establish the connection to the tcp backend, to talk to the visualizer
# zmq_url should match the output of `python run_zmq_server.py`
vis = vis_utils.get_vis(zmq_url="tcp://127.0.0.1:6000")
vis.delete()

# draws a red box in the scene
vis_utils.show_box_test(vis)

In [3]:

with initialize(version_base=None, config_path="../configs"):
    config = compose(config_name="graph_default.yaml")

In [4]:

dataset_inputs = get_dataset_inputs_from_dataset_config(**config.data.dataset_inputs_train, split="train")

In [5]:

# ImageDataset
image_dataset_train = instantiate_from_dict_config(config.data.image_dataset_train, **dataset_inputs.as_dict())

In [6]:

# grab 10 images and show their frustums
indices = random.sample(range(len(image_dataset_train)), k=10)
for idx in indices:
    image = image_dataset_train[idx]["image"]
    camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None)
    pose = camera.get_camera_to_world().double().numpy()
    K = camera.get_intrinsics_matrix().double().numpy()
    vis_utils.draw_camera_frustum(
        vis,
        image=(image.double().numpy() * 255.0),
        pose=pose,
        K=K,
        height=1.0,
        name="{:06d}".format(idx),
        displayed_focal_length=0.5,
        realistic=False,
    )

In [7]:

# move a camera around
num_cameras = len(dataset_inputs.intrinsics)
intrinsics = dataset_inputs.intrinsics
camera_to_world = dataset_inputs.camera_to_world
idx0, idx1 = random.sample(range(num_cameras), k=2)
camera_a = get_camera(intrinsics[idx0], camera_to_world[idx0], None)
camera_b = get_camera(intrinsics[idx1], camera_to_world[idx1], None)

num_steps = 100
fps = 30
estimated_time = num_steps / fps
print("estimated_time:", estimated_time)

camera_path = InterpolatedCameraPath(camera_a, camera_b)
cameras = camera_path.get_path(steps=num_steps)

start = time.time()
for camera in cameras:
    vis_utils.set_camera(vis, camera)
    time.sleep(1 / fps)
time_elapsed = time.time() - start
print("time_elapsed:", time_elapsed)

estimated_time: 3.3333333333333335
time_elapsed: 3.5769083499908447

In [ ]: