nerfstudio/notebooks/visualize_viewer.ipynb

Notebook to visualize the viewer functionality

In [1]:

import os

os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "4"

In [2]:

from notebook_imports import *

import time
import torch
from torch.utils.data import DataLoader

from pyrad.data.dataloader import TrainDataloader
from pyrad.data.image_dataset import ImageDataset, PanopticImageDataset
from pyrad.data.image_sampler import CacheImageSampler
from pyrad.data.pixel_sampler import PixelSampler
from pyrad.data.utils import DatasetInputs, get_dataset_inputs_from_dataset_config
from pyrad.graphs.modules.ray_generator import RayGenerator
from pyrad.graphs.modules.scene_colliders import SceneBoundsCollider, AABBBoxCollider
from pyrad.cameras.cameras import get_camera
from pyrad.cameras.camera_paths import InterpolatedCameraPath
from pyrad.cameras.rays import RayBundle
from pyrad.utils.io import get_absolute_path
from pyrad.utils.plotly import get_line_segments_from_lines
from pyrad.cameras.cameras import get_camera_model
from pyrad.utils.misc import get_dict_to_torch, instantiate_from_dict_config
from pyrad.engine.trainer import Trainer
from pyrad.utils.config import Config, setup_config

# the new import
from pyrad.viewer.backend import vis_utils


from hydra import compose, initialize
from omegaconf import open_dict
import pprint
from tqdm import tqdm
import random

import umsgpack
from pyrad.viewer.backend.utils import get_chunks, get_intrinsics_matrix_and_camera_to_world_h

In [3]:

with initialize(version_base=None, config_path="../configs"):
    config = compose(
        config_name="graph_instant_ngp.yaml",
        overrides=[
            "trainer.resume_train.load_dir=/shared/ethanweber/pyrad/outputs/blender_lego/instant_ngp/2022-07-05_200513/checkpoints",
            "trainer.resume_train.load_step=5000",
            "viewer.render_image_height=50",
        ],
    )
    config = setup_config(config)

In [4]:

trainer = Trainer(config, local_rank=0, world_size=1)
trainer.setup()
trainer.draw_scene_in_viewer()

2022-07-05 23:33:06.262342: I tensorflow/core/util/util.cc:169] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.

In [5]:

dataset_inputs = get_dataset_inputs_from_dataset_config(**config.data.dataset_inputs_train, split="train")

In [6]:

# ImageDataset
image_dataset_train = instantiate_from_dict_config(config.data.image_dataset_train, **dataset_inputs.as_dict())

In [13]:

# move a camera around
num_cameras = len(dataset_inputs.intrinsics)
intrinsics = dataset_inputs.intrinsics
camera_to_world = dataset_inputs.camera_to_world
idx0, idx1 = random.sample(range(num_cameras), k=2)
camera_a = get_camera(intrinsics[idx0], camera_to_world[idx0], None)
camera_b = get_camera(intrinsics[idx1], camera_to_world[idx1], None)

num_steps = 100
fps = 30
estimated_time = num_steps / fps
print("estimated_time:", estimated_time)

camera_path = InterpolatedCameraPath(camera_a, camera_b)
cameras = camera_path.get_path(steps=num_steps)

start = time.time()
for camera in cameras:
    vis_utils.set_camera(trainer.vis, camera)

    # render an image
    outputs = trainer.render_image_in_viewer()

    time.sleep(1 / fps)
    # break
time_elapsed = time.time() - start
print("time_elapsed:", time_elapsed)

estimated_time: 3.3333333333333335
time_elapsed: 14.046947002410889

In [15]:

outputs = trainer.render_image_in_viewer()

In [ ]: