doc refactoring (#267)

* reformatted docs; moved visualizer section; added references to top; hiding some doctrees etc.

* changed realtime viewer tab
This commit is contained in:
evonneng authored and GitHub committed 2022-08-10 16:15:44 -07:00
1 parent 841e03a9a3
commit 3cb44f3a8e
25 files changed
+884 -761

No files matched your search

+521
View File
@@ -0,0 +1,521 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "3c00934d-0f14-4c1a-a563-94d627b377e5",
"metadata": {},
"source": [
"# Programming the viewer\n",
"\n",
"The real-time web viewer enables communication between Python and the ReactJS Client Viewer Application. Here we provide a walkthrough for how to use the viewer to draw information as well as read information from the GUI.\n"
]
},
{
"cell_type": "code",
"execution_count": 72,
"id": "b15dfdc1-5054-40bc-9631-4c7426eecfd9",
"metadata": {
"tags": [
"remove-cell"
]
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The autoreload extension is already loaded. To reload it, use:\n",
" %reload_ext autoreload\n"
]
}
],
"source": [
"# HIDDEN\n",
"%load_ext autoreload\n",
"%autoreload 2\n",
"import pprint\n",
"import torch\n",
"import numpy as np\n",
"import torch\n",
"import mediapy as media\n",
"from nerfactory.data.structs import SceneBounds\n",
"from nerfactory.cameras.cameras import PinholeCamera\n",
"import time"
]
},
{
"cell_type": "markdown",
"id": "a8709c84",
"metadata": {},
"source": [
"## Open the Nerfviewer client\n",
"\n",
"Open the Nerfviewer client by visiting http://nerfviewer.plenoptix.ai/ or running the code locally. *We refer to this document for running the code locally.* You should see a GUI like the following:\n",
"\n",
"```{image} imgs/nerfviewer.png\n",
":align: center\n",
":class: only-light\n",
":width: 600\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "052af8ce",
"metadata": {},
"source": [
"## Start the Bridge Server\n",
"\n",
"Run the following code in the repository home directory\n",
"\n",
"```\n",
"python scripts/run_viewer_bridge_server.py\n",
"```\n",
"\n",
"This will start the bridge server that both the Python code and Client Viewer App will connect to.\n",
"\n",
"Verify that the Client Viewer App is connected."
]
},
{
"cell_type": "code",
"execution_count": 73,
"id": "189b2282",
"metadata": {},
"outputs": [],
"source": [
"# from IPython.display import HTML\n",
"\n",
"# url=\"http://localhost:4000/?localhost:18051\"\n",
"# height = 500\n",
"\n",
"# HTML(\"\"\"\n",
"# <div style=\"height: {height}px; width: 100%; overflow-x: auto; overflow-y: hidden; resize: both\">\n",
"# <iframe src=\"{url}\" style=\"width: 100%; height: 100%; border: none\"></iframe>\n",
"# </div>\n",
"# \"\"\".format(url=url, height=height))"
]
},
{
"cell_type": "markdown",
"id": "46c89306",
"metadata": {},
"source": [
"## Communicate with the Client Viewer App from Python\n",
"\n",
"Run the following code in the repository home directory\n",
"\n",
"```\n",
"python scripts/run_viewer_bridge_server.py\n",
"```\n",
"\n",
"This will start the bridge server that both the Python code and Client Viewer App will connect to.\n",
"\n",
"Verify that the Client Viewer App is connected."
]
},
{
"cell_type": "code",
"execution_count": 74,
"id": "8919ad68",
"metadata": {},
"outputs": [],
"source": [
"# connect to the viewer from Python\n",
"from nerfactory.viewer.server.visualizer import Viewer\n",
"\n",
"vis = Viewer(zmq_url=\"tcp://127.0.0.1:6000\")"
]
},
{
"cell_type": "code",
"execution_count": 75,
"id": "605e0131-bfbe-4309-b338-c49a46923d8f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/sceneState/sceneBounds\n"
]
}
],
"source": [
"# draw scene bounds in the viewer\n",
"s = 1\n",
"scene_bounds = SceneBounds(aabb=torch.tensor([[-s, -s, -s], [s, s, s]]))\n",
"scene_bounds_json = scene_bounds.to_json()\n",
"\n",
"# draw scene bounds in the viewer\n",
"vis[\"sceneState/sceneBounds\"].write(scene_bounds_json)\n",
"\n",
"# draw scene bounds in plotly\n",
"import plotly.graph_objects as go\n",
"\n",
"# TODO:"
]
},
{
"cell_type": "code",
"execution_count": 76,
"id": "fa0acb3c",
"metadata": {
"tags": [
"remove-cell"
]
},
"outputs": [],
"source": [
"# HIDDEN\n",
"def get_random_rotation_matrix():\n",
" roll = torch.randn(1)\n",
" yaw = torch.randn(1)\n",
" pitch = torch.randn(1)\n",
"\n",
" tensor_0 = torch.zeros(1)\n",
" tensor_1 = torch.ones(1)\n",
"\n",
" RX = torch.stack(\n",
" [\n",
" torch.stack([tensor_1, tensor_0, tensor_0]),\n",
" torch.stack([tensor_0, torch.cos(roll), -torch.sin(roll)]),\n",
" torch.stack([tensor_0, torch.sin(roll), torch.cos(roll)]),\n",
" ]\n",
" ).reshape(3, 3)\n",
"\n",
" RY = torch.stack(\n",
" [\n",
" torch.stack([torch.cos(pitch), tensor_0, torch.sin(pitch)]),\n",
" torch.stack([tensor_0, tensor_1, tensor_0]),\n",
" torch.stack([-torch.sin(pitch), tensor_0, torch.cos(pitch)]),\n",
" ]\n",
" ).reshape(3, 3)\n",
"\n",
" RZ = torch.stack(\n",
" [\n",
" torch.stack([torch.cos(yaw), -torch.sin(yaw), tensor_0]),\n",
" torch.stack([torch.sin(yaw), torch.cos(yaw), tensor_0]),\n",
" torch.stack([tensor_0, tensor_0, tensor_1]),\n",
" ]\n",
" ).reshape(3, 3)\n",
"\n",
" R = torch.mm(RZ, RY)\n",
" R = torch.mm(R, RX)\n",
" return R"
]
},
{
"cell_type": "code",
"execution_count": 77,
"id": "88b145d3",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/sceneState/cameras\n"
]
},
{
"data": {
"text/html": [
"<table class=\"show_images\" style=\"border-spacing:0px;\"><tr><td style=\"padding:1px;\"><img width=\"256\" height=\"256\" style=\"image-rendering:pixelated; object-fit:cover;\" src=\"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQAAAAEACAIAAADTED8xAAEAAElEQVR4nAD/fwCAAcghnWl4zNfzhXcWlsCtBTSGpbjInGfr1sCE/o/OR7ogE3POZG34K0kPEJKM4D2iZ1df8QlLE88r0UmnH67Rr2XqaNvZZzsnvWpDucPQPjoSZ+cngkPgJX4PEnVH7ahMvv9uO8LFrSc8Tg7AWjrHuk/MOEItsmrDvtnxuv8R3Z5IOucKCSDOXObvqpLV7JTpH65lA605lklWrvUH51k4qnGeAF3r4DaKwubewGSX1BiOweWpHqUfcJIosrqw4h27DdfgcUD0luOdzNesjSU/hMLl9Up72SrO470HO/bwWgMkZGlcZ0nmYkjP1+jGvzN6hrO0+sdjrIgw+po8NmRwoC2+/iCnthLDb4zAl9Boxl0B/enfvM7mVgt9y3Df2eLpczlD6ufONVFyVddGeyxk2YIQgR0/+FzaonqOYrKhGSfAy01D6xRigvGWzpxw6zCrbbCjGcyLgmejHaAbioAXxgvA3BdLJLi8aK5w1CBLl7bqMuHAxESTtp02hsz9CL2CXXeBIeDJSzIhO53YUYwHF20rh9psvZn6xVbPyLW81hke6SilTzeU7M3kXUJHiwVDiE8ztcTGSKiFxH+t/8oAkbMU+nu0jWLUB6JyKUgSwJMt4jBogL+s7nUXBOAM5yPRNFtkvHWX8XH73HJHiRg2PcSZy0x7yT9wIURamMfUirRxoLGRKpZSl/GKWN/tG6yIVsW89fbqwZByPjuRHBHfIWI3BGXVXiqBsPvUHP55fxcYZE3/yWCGcuPf8w3L5M4Dd+L0ZQw4ejQAvc2Cm9qtiMU7bJ72d+cyJbSXamlFNa9G75pHuXmhNCeyN3374is/tCKxPXDK1czFgktltHfTg/6d7DYYx6AXzHh420c2BV2FZjXkFo5wXt3OWtClVxoYZTGwEEFaOfldZSIvlTxV20j5eQYWzdVXTgBOquUFZQZJGFqsLzxSFvi99fmcAp4S/r85aK3NjnV3eB/3LlFLkyN13IMp/WJl+s1w3bAWKqtSFoNlWuKeWFZOYEiE12/PNQGqmGDKFLR4OAamjb2fQXR3Ve0i8qPSuLAT7wtjv0NDsWQfhgCaD+Ce2PRLfnT4otZdDcgLo0d9lo8QK2TMmGRXm9LQe92jMiCP5hgw/mCRO61vAEyBDAMr7w/3EtXMqlZfDp5lBW12+f+LEFuGgBBUojGJWvGWbWgAF5K7j+X8cNhWguehvBcBFSbmfdlLHcz9W0GHQVZIrSC3bawC/Vi+DvLnR5wH8ouxTMA5kXQk/640mnWQoilxa49OuL3L2ePSQYQtbOIdL7ZGS9Z1Wdo+A/vKhqr0mJ+R2iy3JYLgq0t87Z2lGh+/iM6wjPKpmwyAoQtRkrFWAy/bGypvzqzEy2Q6owwn4pla58wADixTWnvLOOA0prylwlQuWa+fq8XG5q7qXpWsCuv56uKnEgz68xVH33Svc44u6kz/wIeqpkCS5pXYYVQECJ93rzySly2MHKHAcAajugKMGNz/TSodrQtai9e0DB6k4eXfpH65+Xj7k982iepcM2I/kTLhfNlAVrbCyj9RRtR21e/Fq8lpEwsRovKuzD5mnNedMZ+OMF2UI6EZvnAsplxTyfJCELjMZPRyWO0Ok/lB/L+TphPUK6blyaN91omAGGM/j8vqJ+D1MFX1Y35aauTCI3RRtVjapRUk8qV5RWGOAcOPiIjEJKM0bIolHNf9EebYiNp6jjgf7lk8bdbFbZsJndzAXR7IwbMiGERuwM7vWuvj9bduuDnQ1/RG4BsFpsDXN4GwXCELElnPliJyvWaDZpP/XRwHSx5aupf+T+NlwsPSJbiK+eRdNiG8IpxWf1GeVxZJ0B55VFRFG3hrwAJmsHvrZrs5twg1v+IRYRzPCHMfmN8DsyEwe9JVck3eBP1GLEGh4O+FnU64+F0ntdrK1yIdLsMNDlEiuRSUI+bNRzUDLOsD6DHJOrY9lvr+1sF1TW6J6bud18kBNpazKBffa+phnKkx/YDQRRxBeAuijUeApwDm7adzc0jZOkC/PbwiQ9jyZ14k/Z13WVxdBiX1cC48JSYB2h4gqpI1Dfb2klbDjqYi9SZn7dxC7PADvvictffwdzceQxrLqDuLM/bKyOFvhX89pF4iEBmTceBiJHr8FmnanM7tfR1MUfXaMp3l0Z1D6D2bepxk6vA5jOYBlmhklblGDwRnAJz8tdlhR7QlspymkQLZRBtJxfmY6X2feTsZqtA3on7M7MmKAr4vyEmyp64j518oA5f2uR00vnj1xNb2xdIP/bpnRA5/XgsROjl9K7g7vDQhLSmfWFU1Q+xv3dGK3qg6o0T7jUptSS1M7+g8hz/wkR/bmNGmmCgMLd82sBweiGTGRx8vhPMa9zmMz712A3FzT9nSUHkJqYhUcSXnnRwIk7zj+Y83RHt5aN8QGEPAl6P0FylSBkFaYN7zqTQXAOd1IubVgZ+XcXnovvWxW778QiMcpKjZJXViXVWmygFlomayBdupHarfwxOCCnsEFoE1PKP8bnsQssegJCskCqMOxUN+Ar+p3CzqQGXr7JrrbjyYXpC19mhChrJL8AcURWN9tn1eFaqSiKmJeAA4PWZhwxrfbMoZUDEUIPVH42uvWkrLXx7NJqUDiVv6/rzTXoTpGfSvrH8/TFEFwyb/vbi+2lOlHuT2IsJCz2Api5QiXkJW2indabX3+opZn7caUN598AEFFxBRqa6kABdlhWo4ofSahAgw+UqG6oOqKah3fhL1H0XoAGSygez7iUvKbv3jd38FcNLttltQzS8qM4/RxzDJYe9rppctaudl31fMjGHEQmufSdLqu2fqM8UNReoDhIai4EJ5x9rNO4lykyINpvYQ4TF7Dfuof+pq5Yhv2WkdFKSq1tJ0JA1qrjbfsXkr6HWrnmnRnq0jVAOcgQrS6orvWUshJQjDxpYVnHs9yioh0CHq+trv64bhU4vsGu2FTBEM08RmtdBGqp5JJ7TIxlAzm1M9yDHVZFcrFE0GZ7U33lhMgfx8B6wd8LNcYVdTfVGq8bTJHQP0vhcTO+iV+70uEHnnTpv7afr1MeDPsP1BNSdiA4Yg+WQRX+iLBJ5wy60SJSjDIcTH5hv+ut0ZS6HVfCMk/k2rURubgIstjNM1WMqIIMMTx/jv1aaJ6ebnQ75qZRm364KuAazdJR2w21ZqAWRBq6lrBcJ8pgDQxAJIGuAWU+KhxNvtgZ/doF91AkYDdxlhi6LEpYuVOIoL6wCfQn5RQ87sPuMkgkM/H+FyYxFpt3SV810xVid2BC1MT1maFNJNt8ekrw91QUT9CcuLW2LPkXMDpSewDJun1Fl/1BY0uVnyL6n+PRp2pAwrRyYB6G15oEKGC56wTjgrs7Rkc4iUNXKfBoXdPJkYyx6ICtAYS3uWOpMpt1Exdk/JjatQLFFe+jNp/kwttSLoGgeIZ4q/pOuHgH1NPM66TJ/cBWFL5+ulNHhhyFJ2eX7dpIp+LJfKMceqNETPrWcrWGkZ+sS9Ox1EVuNscpeWN1hIAr8lw93UoECxOR8U3+nQ9Qs32Xtoa04LSFJp0qGFmSsAzNxT7rFsvuHC4SsEcsh9Az6QSWMbdiUVavvTJuZh4omv/BOtD7UE5vFq3qkfoX2rytMlKRCFq+8mFKv2Is30Qyb6hSdbJ9BA8J6qvJ0MBjsq5XkBKr89W7SYLYWkVsaZEBgBX8s70xmkKX0YVSVC64O9L8TzsuoZsgn8uxX5PR1HLwoS/6SWFeUM3zNh7Xzmb3C7ThhkWjSAOHMQyqyYD2wg7YzdeDnPtOYFrCVmnM8iM7vBGaRDApIuk8XBrZSdWcJ3lNaa2+ZUPg76Zu1cTovks3Y86AzSslI60eomROvS3n1keJz1WPqukrBUg1Ro5tblyYIRsDNIR/yoYizWTmp/ArfNY48V5yyCDnfusPvycJ/+QMgOXMHpEuMI7+rARyjpARTJLd9FYg+MOMr6VtbnMbJk/U9dlWKzE4m/HDR3U12yekoWMy4s/7URVFp1l/bX8Qm3hD6dUVbO/rfoNeN8Z3Zicvp5KWJwAv1V+4jkY2+hIKVWX5oNlfuthU+PUs2bR4rhxeYbKyBcy9cYh2nsnq7SKHT7eiZhFAJCHVLSz48H+GwbCdGay04kCTAvEucYeVsPG1lxAqa/Px0dwFPPjbi4ct5HlSe82PwK1a2XYrC1Hi3Z2QKEbwL82RYPTd1EYJbLezQ9v7NLbamUI/G5FX92nFU/uej7griwv2oZV04sLnPS6T61Njb2pE9OiakNJeRGZrbQ50kI4bxlXZ0o6AiDrxUDK7R67R7Zg9rG9AJS+wt8nGeqwRWu3YhzAPWborfHQuHN6g27JkvOx3nY+eHw//q26oojuug7TU8wCTgcTLDgo6HZRUOTJ187BhUJsYr56RgXL0nJGEnSb+7GquxmVBWjhQdKbpLzEcJcJLNJgEwYxFV55PCOEqBDvYtGHTmgUrP1Zrv3db4eINXSM6mK+OExOnI3yewFfh75uq+582H9ORYKmqtwDAgRfu9B7YdRwR4n1itWo7x9M9oEdaQV3uaOgUzyOXxLibc87fFpDCytIGXi110WvTvDooRTFxUtOQpx/CLy1R71pcbE4J9ErRCgZIDOf0bd4L8E4eWWiMiT/AqAvExDoV7e+udBtwWQo7rttpyoIns9go5vAqyxWWxSeH+hat03yg6Ag0lTLEXBFqYf5FOGCpfofc9AAwc1vtdNv3VbLine truncated
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"/sceneState/cameras/000000\n",
"/sceneState/cameras/000001\n"
]
},
{
"data": {
"text/plain": [
"b'ok'"
]
},
"execution_count": 77,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# delete any current cameras\n",
"vis[f\"sceneState/cameras\"].delete()\n",
"\n",
"# draw a camera with with an image\n",
"image = torch.rand(256, 256, 3)\n",
"media.show_image(image)\n",
"camera = PinholeCamera(cx=100, cy=100, fx=50, fy=50)\n",
"camera_json = camera.to_json(image=image)\n",
"vis[f\"sceneState/cameras/000000\"].write(camera_json)\n",
"\n",
"# draw a camera with rotation and translation (with a black image)\n",
"image = torch.zeros(256, 256, 3)\n",
"c2w = torch.eye(4)\n",
"c2w[:3, :3] = get_random_rotation_matrix()\n",
"random_translation = torch.rand(1) * 5\n",
"c2w[:3, 3] = random_translation\n",
"camera = PinholeCamera(cx=100, cy=100, fx=50, fy=50, camera_to_world=c2w)\n",
"camera_json = camera.to_json(image=image)\n",
"vis[f\"sceneState/cameras/000001\"].write(camera_json)"
]
},
{
"cell_type": "code",
"execution_count": 78,
"id": "18991f81",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/sceneState/cameras/000000\n"
]
},
{
"data": {
"text/plain": [
"b'ok'"
]
},
"execution_count": 78,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# delete a camera from the Nerfviewer\n",
"vis[\"sceneState/cameras/000000\"].delete()"
]
},
{
"cell_type": "markdown",
"id": "6b798e26",
"metadata": {},
"source": [
"## Read from the GUI sliders"
]
},
{
"cell_type": "code",
"execution_count": 79,
"id": "31cd0de1",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"80\n"
]
}
],
"source": [
"# read the field of view\n",
"camera = vis[f\"renderingState/camera\"].read()\n",
"fov = camera[\"object\"][\"fov\"]\n",
"print(fov)"
]
},
{
"cell_type": "code",
"execution_count": 86,
"id": "feb83aaa",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/renderingState/field_of_view\n"
]
},
{
"data": {
"text/plain": [
"b'ok'"
]
},
"execution_count": 86,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vis[f\"renderingState/field_of_view\"].write(40)"
]
},
{
"cell_type": "code",
"execution_count": 81,
"id": "1abb93e5",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"None\n"
]
}
],
"source": [
"fov = vis[f\"renderingState/camera/object/fov\"].read()\n",
"print(fov)"
]
},
{
"cell_type": "code",
"execution_count": 82,
"id": "a0304523",
"metadata": {},
"outputs": [],
"source": [
"# set the field of view\n",
"camera = vis[f\"renderingState/camera\"].read()\n",
"fov = camera[\"object\"][\"fov\"]"
]
},
{
"cell_type": "code",
"execution_count": 83,
"id": "b3ba0ee7",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"/renderingState/minResolution\n"
]
},
{
"data": {
"text/plain": [
"b'ok'"
]
},
"execution_count": 83,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"vis[f\"renderingState/minResolution\"].write(20)"
]
},
{
"cell_type": "code",
"execution_count": 84,
"id": "bba59227",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"20"
]
},
"execution_count": 84,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# read from a slider\n",
"vis[f\"renderingState/minResolution\"].read()"
]
},
{
"cell_type": "code",
"execution_count": 87,
"id": "fe922185",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'depth'"
]
},
"execution_count": 87,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"# read from output choice\n",
"vis[\"renderingState/output_choice\"].read()"
]
},
{
"cell_type": "code",
"execution_count": 85,
"id": "75c37760",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"['rgb', 'accumulation', 'depth', 'alive_ray_mask']\n",
"None\n"
]
}
],
"source": [
"# read from a dropdown\n",
"output_options = vis[f\"renderingState/output_options\"].read()\n",
"print(output_options)\n",
"output_choice = vis[f\"renderingState/output_choice\"].read()\n",
"print(output_choice)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "98aaa12c",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3.8.13 ('nerfactory')",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.13"
},
"vscode": {
"interpreter": {
"hash": "100252972d4ee5286ab455ef1c778abde05b4affca9d1be78633d3797abf7e8c"
}
}
},
"nbformat": 4,
"nbformat_minor": 5
}