diff --git a/notebooks/visualize_viewer.ipynb b/notebooks/visualize_viewer.ipynb new file mode 100644 index 000000000..8ae7870f0 --- /dev/null +++ b/notebooks/visualize_viewer.ipynb @@ -0,0 +1,197 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3c00934d-0f14-4c1a-a563-94d627b377e5", + "metadata": {}, + "source": [ + "## Notebook to visualize the viewer functionality\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b15dfdc1-5054-40bc-9631-4c7426eecfd9", + "metadata": {}, + "outputs": [], + "source": [ + "from notebook_imports import *\n", + "\n", + "import time\n", + "import torch\n", + "from torch.utils.data import DataLoader\n", + "\n", + "from pyrad.data.dataloader import TrainDataloader\n", + "from pyrad.data.image_dataset import ImageDataset, PanopticImageDataset\n", + "from pyrad.data.image_sampler import CacheImageSampler\n", + "from pyrad.data.pixel_sampler import PixelSampler\n", + "from pyrad.data.utils import DatasetInputs, get_dataset_inputs_from_dataset_config\n", + "from pyrad.graphs.modules.ray_generator import RayGenerator\n", + "from pyrad.graphs.modules.scene_colliders import SceneBoundsCollider, AABBBoxCollider\n", + "from pyrad.cameras.cameras import get_camera\n", + "from pyrad.cameras.camera_paths import InterpolatedCameraPath\n", + "from pyrad.cameras.rays import RayBundle\n", + "from pyrad.utils.io import get_absolute_path\n", + "from pyrad.utils.plotly import get_line_segments_from_lines\n", + "from pyrad.cameras.cameras import get_camera_model\n", + "from pyrad.utils.misc import get_dict_to_torch, instantiate_from_dict_config\n", + "\n", + "# the new import\n", + "from pyrad.viewer.backend import vis_utils\n", + "\n", + "\n", + "from hydra import compose, initialize\n", + "from omegaconf import open_dict\n", + "import pprint\n", + "from tqdm import tqdm\n", + "import random" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "df82e083-e261-43e0-a1dc-05c6a59d16e1", + "metadata": {}, + "outputs": [], + "source": [ + "# establish the connection to the tcp backend, to talk to the visualizer\n", + "# zmq_url should match the output of `python run_zmq_server.py`\n", + "vis = vis_utils.get_vis(zmq_url=\"tcp://127.0.0.1:6000\")\n", + "vis.delete()\n", + "\n", + "# draws a red box in the scene\n", + "vis_utils.show_box_test(vis)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "7fe7478d", + "metadata": {}, + "outputs": [], + "source": [ + "with initialize(version_base=None, config_path=\"../configs\"):\n", + " config = compose(config_name=\"graph_default.yaml\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "642b0a25-0e88-4eac-b661-a57a74fb10fc", + "metadata": {}, + "outputs": [], + "source": [ + "dataset_inputs = get_dataset_inputs_from_dataset_config(**config.data.dataset_inputs_train, split=\"train\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "938f090b-3b28-4fd6-82dc-d4b5c5917134", + "metadata": {}, + "outputs": [], + "source": [ + "# ImageDataset\n", + "image_dataset_train = instantiate_from_dict_config(config.data.image_dataset_train, **dataset_inputs.as_dict())" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "5d014a6b-50fa-48ab-b135-0ab2b81c7529", + "metadata": {}, + "outputs": [], + "source": [ + "# grab 10 images and show their frustums\n", + "indices = random.sample(range(len(image_dataset_train)), k=10)\n", + "for idx in indices:\n", + " image = image_dataset_train[idx][\"image\"]\n", + " camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None)\n", + " pose = camera.get_camera_to_world().double().numpy()\n", + " K = camera.get_intrinsics_matrix().double().numpy()\n", + " vis_utils.draw_camera_frustum(\n", + " vis,\n", + " image=(image.double().numpy() * 255.0),\n", + " pose=pose,\n", + " K=K,\n", + " height=1.0,\n", + " name=\"{:06d}\".format(idx),\n", + " displayed_focal_length=0.5,\n", + " realistic=False,\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "fd04703a-aa50-45e4-95fd-4d881ff18d61", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "estimated_time: 3.3333333333333335\n", + "time_elapsed: 3.5769083499908447\n" + ] + } + ], + "source": [ + "# move a camera around\n", + "num_cameras = len(dataset_inputs.intrinsics)\n", + "intrinsics = dataset_inputs.intrinsics\n", + "camera_to_world = dataset_inputs.camera_to_world\n", + "idx0, idx1 = random.sample(range(num_cameras), k=2)\n", + "camera_a = get_camera(intrinsics[idx0], camera_to_world[idx0], None)\n", + "camera_b = get_camera(intrinsics[idx1], camera_to_world[idx1], None)\n", + "\n", + "num_steps = 100\n", + "fps = 30\n", + "estimated_time = num_steps / fps\n", + "print(\"estimated_time:\", estimated_time)\n", + "\n", + "camera_path = InterpolatedCameraPath(camera_a, camera_b)\n", + "cameras = camera_path.get_path(steps=num_steps)\n", + "\n", + "start = time.time()\n", + "for camera in cameras:\n", + " vis_utils.set_camera(vis, camera)\n", + " time.sleep(1 / fps)\n", + "time_elapsed = time.time() - start\n", + "print(\"time_elapsed:\", time_elapsed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f1cd7183-c514-4f87-a8cc-e14228228e24", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "interpreter": { + "hash": "4a749acc7d255f078aee52e0584cc77b3cb5aaed1b3a7407ec4262c1bf6cb526" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}