diff --git a/configs/graph_default.yaml b/configs/graph_default.yaml index 8cec6a75a..7e2bfa158 100644 --- a/configs/graph_default.yaml +++ b/configs/graph_default.yaml @@ -98,3 +98,10 @@ optimizers: _target_: pyrad.optimizers.optimizers.ExponentialDecaySchedule lr_final: 0.000005 max_steps: 1000000 + +viewer: + enable: False + zmq_url: tcp://127.0.0.1:6000 + render_image_height: 100 + steps_per_render_image: 100 + num_rays_per_chunk: 4096 diff --git a/configs/graph_instant_ngp.yaml b/configs/graph_instant_ngp.yaml index 924c50c0a..63bd04190 100644 --- a/configs/graph_instant_ngp.yaml +++ b/configs/graph_instant_ngp.yaml @@ -23,3 +23,9 @@ optimizers: optimizer: lr: 1e-2 scheduler: null + +viewer: + enable: True + render_image_height: 200 + steps_per_render_image: 5 + num_rays_per_chunk: 16384 \ No newline at end of file diff --git a/environment/requirements.txt b/environment/requirements.txt index 39bca324e..bb53e7a34 100644 --- a/environment/requirements.txt +++ b/environment/requirements.txt @@ -1,3 +1,6 @@ +aiohttp +aiortc +av black[jupyter]==22.3.0 functorch==0.1.1 h5py>=2.9.0 diff --git a/notebooks/visualize_viewer.ipynb b/notebooks/visualize_viewer.ipynb index 8ae7870f0..f1f71238b 100644 --- a/notebooks/visualize_viewer.ipynb +++ b/notebooks/visualize_viewer.ipynb @@ -10,7 +10,20 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, + "id": "8501e01d-f932-4a75-8259-2def67a546d7", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n", + "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "b15dfdc1-5054-40bc-9631-4c7426eecfd9", "metadata": {}, "outputs": [], @@ -35,6 +48,8 @@ "from pyrad.utils.plotly import get_line_segments_from_lines\n", "from pyrad.cameras.cameras import get_camera_model\n", "from pyrad.utils.misc import get_dict_to_torch, instantiate_from_dict_config\n", + "from pyrad.engine.trainer import Trainer\n", + "from pyrad.utils.config import Config, setup_config\n", "\n", "# the new import\n", "from pyrad.viewer.backend import vis_utils\n", @@ -44,19 +59,31 @@ "from omegaconf import open_dict\n", "import pprint\n", "from tqdm import tqdm\n", - "import random" + "import random\n", + "\n", + "import umsgpack\n", + "from pyrad.viewer.backend.utils import get_chunks, get_intrinsics_matrix_and_camera_to_world_h" ] }, { "cell_type": "code", - "execution_count": 2, - "id": "df82e083-e261-43e0-a1dc-05c6a59d16e1", + "execution_count": null, + "id": "032e4c9e-25be-4cc0-9ac8-a5be723b775a", "metadata": {}, "outputs": [], "source": [ "# establish the connection to the tcp backend, to talk to the visualizer\n", "# zmq_url should match the output of `python run_zmq_server.py`\n", - "vis = vis_utils.get_vis(zmq_url=\"tcp://127.0.0.1:6000\")\n", + "vis = vis_utils.get_vis(zmq_url=\"tcp://127.0.0.1:6000\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "df82e083-e261-43e0-a1dc-05c6a59d16e1", + "metadata": {}, + "outputs": [], + "source": [ "vis.delete()\n", "\n", "# draws a red box in the scene\n", @@ -65,18 +92,99 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "7fe7478d", "metadata": {}, "outputs": [], "source": [ "with initialize(version_base=None, config_path=\"../configs\"):\n", - " config = compose(config_name=\"graph_default.yaml\")" + " config = compose(\n", + " config_name=\"graph_default.yaml\",\n", + " overrides=[\n", + " \"trainer.resume_train.load_dir=/shared/ethanweber/pyrad/outputs/blender_lego/vanilla_nerf/2022-07-04_221907/checkpoints\",\n", + " \"trainer.resume_train.load_step=4000\",\n", + " ],\n", + " )\n", + " config = setup_config(config)" ] }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, + "id": "5d231e36-dd38-489b-a3a3-860cdf49e551", + "metadata": {}, + "outputs": [], + "source": [ + "trainer = Trainer(config, local_rank=0, world_size=1)\n", + "trainer.setup()\n", + "trainer.draw_scene_in_viewer()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1aba9ecb-0022-4896-833d-577abe532d23", + "metadata": {}, + "outputs": [], + "source": [ + "aabb = trainer.dataset_inputs_train.scene_bounds.aabb\n", + "vis_utils.draw_aabb(trainer.vis, aabb, name=\"dataset_inputs_train/scene_bounds/aabb\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b23fc89-bfc3-40c4-abba-fad1317aeb21", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5a829d31-4fc6-4908-963d-7326d3aa52da", + "metadata": {}, + "outputs": [], + "source": [ + "outputs = trainer.render_image_in_viewer()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84bd64de-2e0c-458e-a4b7-1084f1cca1c5", + "metadata": {}, + "outputs": [], + "source": [ + "trainer.vis.delete()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ed9cdfa-6359-4a70-95c7-033e853e9b50", + "metadata": {}, + "outputs": [], + "source": [ + "while True:\n", + " outputs = trainer.render_image_in_viewer()\n", + " image = outputs[\"rgb_fine\"].cpu().numpy().astype(\"float64\") * 255\n", + " trainer.vis[\"/Cameras/Main Camera\"].set_image(image)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3bcd221-05a8-494e-b817-3e2de7f5a0df", + "metadata": {}, + "outputs": [], + "source": [ + "image.shape" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "642b0a25-0e88-4eac-b661-a57a74fb10fc", "metadata": {}, "outputs": [], @@ -86,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "938f090b-3b28-4fd6-82dc-d4b5c5917134", "metadata": {}, "outputs": [], @@ -97,45 +205,10 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "5d014a6b-50fa-48ab-b135-0ab2b81c7529", - "metadata": {}, - "outputs": [], - "source": [ - "# grab 10 images and show their frustums\n", - "indices = random.sample(range(len(image_dataset_train)), k=10)\n", - "for idx in indices:\n", - " image = image_dataset_train[idx][\"image\"]\n", - " camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None)\n", - " pose = camera.get_camera_to_world().double().numpy()\n", - " K = camera.get_intrinsics_matrix().double().numpy()\n", - " vis_utils.draw_camera_frustum(\n", - " vis,\n", - " image=(image.double().numpy() * 255.0),\n", - " pose=pose,\n", - " K=K,\n", - " height=1.0,\n", - " name=\"{:06d}\".format(idx),\n", - " displayed_focal_length=0.5,\n", - " realistic=False,\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 7, + "execution_count": null, "id": "fd04703a-aa50-45e4-95fd-4d881ff18d61", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "estimated_time: 3.3333333333333335\n", - "time_elapsed: 3.5769083499908447\n" - ] - } - ], + "outputs": [], "source": [ "# move a camera around\n", "num_cameras = len(dataset_inputs.intrinsics)\n", diff --git a/pyrad/engine/trainer.py b/pyrad/engine/trainer.py index 8853c6b15..2be34c032 100644 --- a/pyrad/engine/trainer.py +++ b/pyrad/engine/trainer.py @@ -17,13 +17,16 @@ Code to train model. """ import logging import os +import random from typing import Dict import torch import torch.distributed as dist +import umsgpack from torch.nn.parallel import DistributedDataParallel as DDP from torchtyping import TensorType +from pyrad.cameras.cameras import get_camera, get_intrinsics_from_intrinsics_matrix from pyrad.cameras.rays import RayBundle from pyrad.data.dataloader import EvalDataloader, setup_dataset_eval, setup_dataset_train from pyrad.graphs.base import setup_graph @@ -32,6 +35,9 @@ from pyrad.utils import profiler, writer from pyrad.utils.config import Config from pyrad.utils.decorators import check_main_thread from pyrad.utils.writer import EventName, TimeWriter +from pyrad.viewer.backend import vis_utils +from pyrad.viewer.backend.utils import get_intrinsics_matrix_and_camera_to_world_h +from pyrad.viewer.backend.visualizer import ViewerWindow, Visualizer logging.getLogger("PIL").setLevel(logging.WARNING) @@ -40,10 +46,9 @@ class Trainer: """Training class Args: - config (Config): _description_ - local_rank (int, optional): _description_. Defaults to 0. - world_size (int, optional): _description_. Defaults to 1. - cpu (bool, optional): Whether or not to use the CPU. + config (Config): The configuration object. + local_rank (int, optional): Local rank of the process. Defaults to 0. + world_size (int, optional): World size of the process. Defaults to 1. """ def __init__(self, config: Config, local_rank: int = 0, world_size: int = 1): @@ -52,6 +57,7 @@ class Trainer: self.world_size = world_size self.device = "cpu" if world_size == 0 else f"cuda:{local_rank}" # dataset variables + self.dataset_inputs_train = None self.dataloader_train = None self.dataloader_eval = None # model variables @@ -62,11 +68,20 @@ class Trainer: writer.setup_event_writers(config.logging, max_iter=config.trainer.max_num_iterations) profiler.setup_profiler(config.logging) + self.vis = None + if self.config.viewer.enable: + window = ViewerWindow(zmq_url=self.config.viewer.zmq_url) + self.vis = Visualizer(window=window) + logging.info("Connected to viewer at %s", self.config.viewer.zmq_url) + self.vis.delete() + else: + logging.info("Continuing without viewer.") + def setup(self, test_mode=False): """Setup the Trainer by calling other setup functions.""" - dataset_inputs_train, self.dataloader_train = setup_dataset_train(self.config.data, device=self.device) + self.dataset_inputs_train, self.dataloader_train = setup_dataset_train(self.config.data, device=self.device) _, self.dataloader_eval = setup_dataset_eval(self.config.data, test_mode=test_mode, device=self.device) - self.graph = setup_graph(self.config.graph, dataset_inputs_train, device=self.device) + self.graph = setup_graph(self.config.graph, self.dataset_inputs_train, device=self.device) self.optimizers = setup_optimizers(self.config.optimizers, self.graph.get_param_groups()) if self.config.trainer.resume_train.load_dir: @@ -90,7 +105,11 @@ class Trainer: return loss_sum def train(self) -> None: - """_summary_""" + """Train the model.""" + + if self.vis: + self.draw_scene_in_viewer() + with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME): num_iterations = self.config.trainer.max_num_iterations iter_dataloader_train = iter(self.dataloader_train) @@ -106,6 +125,13 @@ class Trainer: writer.put_dict(name="Loss/train-loss_dict", scalar_dict=loss_dict, step=step) if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0: self._save_checkpoint(self.config.trainer.model_dir, step) + if ( + self.vis + and step != 0 + and self.config.viewer.steps_per_render_image + and step % self.config.viewer.steps_per_render_image == 0 + ): + _ = self.render_image_in_viewer() if step % self.config.trainer.steps_per_test == 0: self.eval_with_dataloader(self.dataloader_eval, step=step) self._write_out_storage(step) @@ -128,7 +154,7 @@ class Trainer: def _load_checkpoint(self) -> None: """helper function to load graph and optimizer from prespecified checkpoint""" - load_config = self.config.resume_train + load_config = self.config.trainer.resume_train load_path = os.path.join(load_config.load_dir, f"step-{load_config.load_step:09d}.ckpt") assert os.path.exists(load_path), f"Checkpoint {load_path} does not exist" loaded_state = torch.load(load_path, map_location="cpu") @@ -200,6 +226,60 @@ class Trainer: self.graph.train() return psnr + def draw_scene_in_viewer(self): + """Draw some images and the scene aabb in the viewer.""" + image_dataset_train = self.dataloader_train.image_sampler.dataset + dataset_inputs = self.dataset_inputs_train + indices = random.sample(range(len(image_dataset_train)), k=10) + for idx in indices: + image = image_dataset_train[idx]["image"] + camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None) + pose = camera.get_camera_to_world().double().numpy() + K = camera.get_intrinsics_matrix().double().numpy() + vis_utils.draw_camera_frustum( + self.vis, + image=(image.double().numpy() * 255.0), + pose=pose, + K=K, + height=1.0, + name=f"image_dataset_train/{idx:06d}", + displayed_focal_length=0.5, + realistic=False, + ) + aabb = dataset_inputs.scene_bounds.aabb + vis_utils.draw_aabb(self.vis, aabb, name="dataset_inputs_train/scene_bounds/aabb") + + @profiler.time_function + def render_image_in_viewer(self): + """ + Draw an image using the current camera pose from the viewer. + The image is sent of a TCP connection and then uses WebRTC to send it to the viewer. + """ + data = self.vis["/Cameras/Main Camera"].get_object() + message = umsgpack.unpackb(data) + camera_object = message["object"]["object"] + image_height = self.config.viewer.render_image_height + intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h( + camera_object, image_height=image_height + ) + + camera_to_world = camera_to_world_h[:3, :] + intrinsics = get_intrinsics_from_intrinsics_matrix(intrinsics_matrix) + camera = get_camera(intrinsics, camera_to_world) + camera_ray_bundle = camera.get_camera_ray_bundle(device=self.device) + camera_ray_bundle.num_rays_per_chunk = self.config.viewer.num_rays_per_chunk + + self.graph.eval() + outputs = self.graph.get_outputs_for_camera_ray_bundle(camera_ray_bundle) + self.graph.train() + + # gross hack to get the image key, depending on which keys the graph uses + rgb_key = "rgb" if "rgb" in outputs else "rgb_fine" + # TODO: make it such that the TCP connection doesn't need float64 + image = outputs[rgb_key].cpu().numpy().astype("float64") * 255 + self.vis["/Cameras/Main Camera"].set_image(image) + return outputs + def eval_with_dataloader(self, dataloader: EvalDataloader, step: int = None) -> None: """Run evaluation with a given dataloader. diff --git a/pyrad/utils/config.py b/pyrad/utils/config.py index 39ba7bc9c..e5ca0622b 100644 --- a/pyrad/utils/config.py +++ b/pyrad/utils/config.py @@ -116,6 +116,7 @@ class Config: data: DataConfig = MISSING graph: GraphConfig = MISSING optimizers: Dict[str, Any] = MISSING + viewer: Dict[str, Any] = MISSING # additional optional parameters here hydra: Optional[Dict[str, Any]] = None @@ -137,6 +138,7 @@ def setup_config(config: DictConfig) -> Config: data = DataConfig(**config.data) graph = GraphConfig(**config.graph) optimizers = config.optimizers + viewer = config.viewer return Config( machine=machine, logging=logging, @@ -146,4 +148,5 @@ def setup_config(config: DictConfig) -> Config: data=data, graph=graph, optimizers=optimizers, + viewer=viewer, ) diff --git a/pyrad/viewer/app/public/index.html b/pyrad/viewer/app/public/index.html index 76b7edd48..6d833f9ff 100644 --- a/pyrad/viewer/app/public/index.html +++ b/pyrad/viewer/app/public/index.html @@ -1,14 +1,12 @@ -
+ + - + -