diff --git a/.github/workflows/viewer_client_app.yml b/.github/workflows/viewer_client_app.yml
index 49d096997..10b889676 100644
--- a/.github/workflows/viewer_client_app.yml
+++ b/.github/workflows/viewer_client_app.yml
@@ -18,7 +18,7 @@ jobs:
uses: actions/checkout@master
- name: Install Node.js
- uses: actions/setup-node@v1
+ uses: actions/setup-node@v3
with:
node-version: 17.8.0
@@ -54,4 +54,4 @@ jobs:
uses: peaceiris/actions-gh-pages@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
- publish_dir: ./nerfactory/viewer/app/build
+ publish_dir: ./nerfactory/viewer/app/build
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 76b54487c..afd28f97a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -121,6 +121,7 @@ celerybeat.pid
# Environments
.env
+.envrc
.venv
env/
venv/
diff --git a/.prettierrc.js b/.prettierrc.js
new file mode 100644
index 000000000..3b519289b
--- /dev/null
+++ b/.prettierrc.js
@@ -0,0 +1,6 @@
+module.exports = {
+ trailingComma: 'all',
+ arrowParens: 'always',
+ singleQuote: true,
+ jsxSingleQuote: false,
+ };
\ No newline at end of file
diff --git a/.vscode/launch.json b/.vscode/launch.json
index 962180731..953a075df 100644
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@@ -4,6 +4,13 @@
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
+ {
+ "name": "Launch Chrome against localhost",
+ "request": "launch",
+ "type": "chrome",
+ "url": "http://localhost:4000",
+ "webRoot": "${workspaceFolder}/nerfactory/viewer/app"
+ },
{
"name": "Python: Current File",
"type": "python",
@@ -25,7 +32,9 @@
"request": "launch",
"program": "scripts/run_train.py",
"console": "integratedTerminal",
- "args": ["machine.num_gpus=0"]
+ "args": [
+ "machine.num_gpus=0"
+ ]
},
{
"name": "Python: run_train ngp (gpu)",
@@ -33,7 +42,9 @@
"request": "launch",
"program": "scripts/run_train.py",
"console": "integratedTerminal",
- "args": ["--config-name=graph_instant_ngp.yaml"]
+ "args": [
+ "--config-name=graph_instant_ngp.yaml"
+ ]
},
{
"name": "Python: run_train mipnerf 360 (gpu)",
@@ -41,7 +52,9 @@
"request": "launch",
"program": "scripts/run_train.py",
"console": "integratedTerminal",
- "args": ["--config-name=graph_mipnerf_360.yaml"]
+ "args": [
+ "--config-name=graph_mipnerf_360.yaml"
+ ]
},
{
"name": "Python: run_train nerfw (gpu)",
@@ -66,4 +79,4 @@
]
}
]
-}
+}
\ No newline at end of file
diff --git a/.vscode/settings.json b/.vscode/settings.json
index 6a295ac34..aadaeb8e8 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -8,6 +8,21 @@
"**/temp*": true
},
"editor.defaultFormatter": "esbenp.prettier-vscode",
+ "[javascript]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[javascriptreact]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[typescript]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "[typescriptreact]": {
+ "editor.defaultFormatter": "esbenp.prettier-vscode"
+ },
+ "prettier.configPath": ".prettierrc.js",
+ "typescript.suggestionActions.enabled": false,
+ "javascript.suggestionActions.enabled": false,
"[python]": {
"editor.defaultFormatter": "ms-python.python",
"editor.codeActionsOnSave": {
@@ -17,11 +32,17 @@
"editor.formatOnSave": true,
"python.envFile": "${workspaceFolder}/.env",
"python.formatting.provider": "black",
- "python.formatting.blackArgs": ["--line-length=120"],
+ "python.formatting.blackArgs": [
+ "--line-length=120"
+ ],
"python.linting.pylintEnabled": true,
"python.linting.flake8Enabled": false,
"python.linting.enabled": true,
- "python.sortImports.args": ["--src=${workspaceFolder}", "--profile", "black"],
+ "python.sortImports.args": [
+ "--src=${workspaceFolder}",
+ "--profile",
+ "black"
+ ],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true,
"python.linting.ignorePatterns": [
@@ -29,6 +50,16 @@
".vscode/*.py",
"external/**/*.py"
],
+ "esbonio.sphinx.confDir": "",
+ // eslint extension options
+ "javascript.validate.enable": false,
+ "eslint.enable": true,
+ "eslint.validate": [
+ "javascript",
+ "javascriptreact",
+ "typescript",
+ "typescriptreact"
+ ],
"C_Cpp.errorSquiggles": "Disabled",
"files.associations": {
"array": "cpp",
@@ -96,4 +127,4 @@
"__node_handle": "cpp"
},
"python.analysis.typeCheckingMode": "basic"
-}
+}
\ No newline at end of file
diff --git a/docs/notebooks/imgs/nerfviewer.png b/docs/notebooks/imgs/nerfviewer.png
new file mode 100644
index 000000000..dd7bcc941
Binary files /dev/null and b/docs/notebooks/imgs/nerfviewer.png differ
diff --git a/docs/notebooks/index.rst b/docs/notebooks/index.rst
index 6279c1d3d..c68884456 100644
--- a/docs/notebooks/index.rst
+++ b/docs/notebooks/index.rst
@@ -11,4 +11,5 @@ Components
visualize_samplers.ipynb
visualize_spatial_distortions.ipynb
visualize_encoders.ipynb
+ visualize_viewer.ipynb
\ No newline at end of file
diff --git a/docs/notebooks/visualize_viewer.ipynb b/docs/notebooks/visualize_viewer.ipynb
new file mode 100644
index 000000000..00313a672
--- /dev/null
+++ b/docs/notebooks/visualize_viewer.ipynb
@@ -0,0 +1,521 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "3c00934d-0f14-4c1a-a563-94d627b377e5",
+ "metadata": {},
+ "source": [
+ "# Visualize with our Viewer\n",
+ "\n",
+ "The real-time web viewer enables communication between Python and the ReactJS Client Viewer Application. Here we provide a walkthrough for how to use the viewer to draw information as well as read information from the GUI.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 72,
+ "id": "b15dfdc1-5054-40bc-9631-4c7426eecfd9",
+ "metadata": {
+ "tags": [
+ "remove-cell"
+ ]
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "The autoreload extension is already loaded. To reload it, use:\n",
+ " %reload_ext autoreload\n"
+ ]
+ }
+ ],
+ "source": [
+ "# HIDDEN\n",
+ "%load_ext autoreload\n",
+ "%autoreload 2\n",
+ "import pprint\n",
+ "import torch\n",
+ "import numpy as np\n",
+ "import torch\n",
+ "import mediapy as media\n",
+ "from nerfactory.data.structs import SceneBounds\n",
+ "from nerfactory.cameras.cameras import PinholeCamera\n",
+ "import time"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "a8709c84",
+ "metadata": {},
+ "source": [
+ "## Open the Nerfviewer client\n",
+ "\n",
+ "Open the Nerfviewer client by visiting http://nerfviewer.plenoptix.ai/ or running the code locally. *We refer to this document for running the code locally.* You should see a GUI like the following:\n",
+ "\n",
+ "```{image} imgs/nerfviewer.png\n",
+ ":align: center\n",
+ ":class: only-light\n",
+ ":width: 600\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "052af8ce",
+ "metadata": {},
+ "source": [
+ "## Start the Bridge Server\n",
+ "\n",
+ "Run the following code in the repository home directory\n",
+ "\n",
+ "```\n",
+ "python scripts/run_viewer_bridge_server.py\n",
+ "```\n",
+ "\n",
+ "This will start the bridge server that both the Python code and Client Viewer App will connect to.\n",
+ "\n",
+ "Verify that the Client Viewer App is connected."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 73,
+ "id": "189b2282",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# from IPython.display import HTML\n",
+ "\n",
+ "# url=\"http://localhost:4000/?localhost:18051\"\n",
+ "# height = 500\n",
+ "\n",
+ "# HTML(\"\"\"\n",
+ "#
\n",
+ "# \n",
+ "#
\n",
+ "# \"\"\".format(url=url, height=height))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "46c89306",
+ "metadata": {},
+ "source": [
+ "## Communicate with the Client Viewer App from Python\n",
+ "\n",
+ "Run the following code in the repository home directory\n",
+ "\n",
+ "```\n",
+ "python scripts/run_viewer_bridge_server.py\n",
+ "```\n",
+ "\n",
+ "This will start the bridge server that both the Python code and Client Viewer App will connect to.\n",
+ "\n",
+ "Verify that the Client Viewer App is connected."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 74,
+ "id": "8919ad68",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# connect to the viewer from Python\n",
+ "from nerfactory.viewer.server.visualizer import Viewer\n",
+ "\n",
+ "vis = Viewer(zmq_url=\"tcp://127.0.0.1:6000\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 75,
+ "id": "605e0131-bfbe-4309-b338-c49a46923d8f",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/sceneState/sceneBounds\n"
+ ]
+ }
+ ],
+ "source": [
+ "# draw scene bounds in the viewer\n",
+ "s = 1\n",
+ "scene_bounds = SceneBounds(aabb=torch.tensor([[-s, -s, -s], [s, s, s]]))\n",
+ "scene_bounds_json = scene_bounds.to_json()\n",
+ "\n",
+ "# draw scene bounds in the viewer\n",
+ "vis[\"sceneState/sceneBounds\"].write(scene_bounds_json)\n",
+ "\n",
+ "# draw scene bounds in plotly\n",
+ "import plotly.graph_objects as go\n",
+ "\n",
+ "# TODO:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 76,
+ "id": "fa0acb3c",
+ "metadata": {
+ "tags": [
+ "remove-cell"
+ ]
+ },
+ "outputs": [],
+ "source": [
+ "# HIDDEN\n",
+ "def get_random_rotation_matrix():\n",
+ " roll = torch.randn(1)\n",
+ " yaw = torch.randn(1)\n",
+ " pitch = torch.randn(1)\n",
+ "\n",
+ " tensor_0 = torch.zeros(1)\n",
+ " tensor_1 = torch.ones(1)\n",
+ "\n",
+ " RX = torch.stack(\n",
+ " [\n",
+ " torch.stack([tensor_1, tensor_0, tensor_0]),\n",
+ " torch.stack([tensor_0, torch.cos(roll), -torch.sin(roll)]),\n",
+ " torch.stack([tensor_0, torch.sin(roll), torch.cos(roll)]),\n",
+ " ]\n",
+ " ).reshape(3, 3)\n",
+ "\n",
+ " RY = torch.stack(\n",
+ " [\n",
+ " torch.stack([torch.cos(pitch), tensor_0, torch.sin(pitch)]),\n",
+ " torch.stack([tensor_0, tensor_1, tensor_0]),\n",
+ " torch.stack([-torch.sin(pitch), tensor_0, torch.cos(pitch)]),\n",
+ " ]\n",
+ " ).reshape(3, 3)\n",
+ "\n",
+ " RZ = torch.stack(\n",
+ " [\n",
+ " torch.stack([torch.cos(yaw), -torch.sin(yaw), tensor_0]),\n",
+ " torch.stack([torch.sin(yaw), torch.cos(yaw), tensor_0]),\n",
+ " torch.stack([tensor_0, tensor_0, tensor_1]),\n",
+ " ]\n",
+ " ).reshape(3, 3)\n",
+ "\n",
+ " R = torch.mm(RZ, RY)\n",
+ " R = torch.mm(R, RX)\n",
+ " return R"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 77,
+ "id": "88b145d3",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/sceneState/cameras\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ " |
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/sceneState/cameras/000000\n",
+ "/sceneState/cameras/000001\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "b'ok'"
+ ]
+ },
+ "execution_count": 77,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# delete any current cameras\n",
+ "vis[f\"sceneState/cameras\"].delete()\n",
+ "\n",
+ "# draw a camera with with an image\n",
+ "image = torch.rand(256, 256, 3)\n",
+ "media.show_image(image)\n",
+ "camera = PinholeCamera(cx=100, cy=100, fx=50, fy=50)\n",
+ "camera_json = camera.to_json(image=image)\n",
+ "vis[f\"sceneState/cameras/000000\"].write(camera_json)\n",
+ "\n",
+ "# draw a camera with rotation and translation (with a black image)\n",
+ "image = torch.zeros(256, 256, 3)\n",
+ "c2w = torch.eye(4)\n",
+ "c2w[:3, :3] = get_random_rotation_matrix()\n",
+ "random_translation = torch.rand(1) * 5\n",
+ "c2w[:3, 3] = random_translation\n",
+ "camera = PinholeCamera(cx=100, cy=100, fx=50, fy=50, camera_to_world=c2w)\n",
+ "camera_json = camera.to_json(image=image)\n",
+ "vis[f\"sceneState/cameras/000001\"].write(camera_json)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 78,
+ "id": "18991f81",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/sceneState/cameras/000000\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "b'ok'"
+ ]
+ },
+ "execution_count": 78,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# delete a camera from the Nerfviewer\n",
+ "vis[\"sceneState/cameras/000000\"].delete()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6b798e26",
+ "metadata": {},
+ "source": [
+ "## Read from the GUI sliders"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 79,
+ "id": "31cd0de1",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "80\n"
+ ]
+ }
+ ],
+ "source": [
+ "# read the field of view\n",
+ "camera = vis[f\"renderingState/camera\"].read()\n",
+ "fov = camera[\"object\"][\"fov\"]\n",
+ "print(fov)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 86,
+ "id": "feb83aaa",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/renderingState/field_of_view\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "b'ok'"
+ ]
+ },
+ "execution_count": 86,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "vis[f\"renderingState/field_of_view\"].write(40)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 81,
+ "id": "1abb93e5",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "None\n"
+ ]
+ }
+ ],
+ "source": [
+ "fov = vis[f\"renderingState/camera/object/fov\"].read()\n",
+ "print(fov)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 82,
+ "id": "a0304523",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# set the field of view\n",
+ "camera = vis[f\"renderingState/camera\"].read()\n",
+ "fov = camera[\"object\"][\"fov\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 83,
+ "id": "b3ba0ee7",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "/renderingState/minResolution\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "b'ok'"
+ ]
+ },
+ "execution_count": 83,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "vis[f\"renderingState/minResolution\"].write(20)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 84,
+ "id": "bba59227",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "20"
+ ]
+ },
+ "execution_count": 84,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# read from a slider\n",
+ "vis[f\"renderingState/minResolution\"].read()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 87,
+ "id": "fe922185",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "'depth'"
+ ]
+ },
+ "execution_count": 87,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# read from output choice\n",
+ "vis[\"renderingState/output_choice\"].read()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 85,
+ "id": "75c37760",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['rgb', 'accumulation', 'depth', 'alive_ray_mask']\n",
+ "None\n"
+ ]
+ }
+ ],
+ "source": [
+ "# read from a dropdown\n",
+ "output_options = vis[f\"renderingState/output_options\"].read()\n",
+ "print(output_options)\n",
+ "output_choice = vis[f\"renderingState/output_choice\"].read()\n",
+ "print(output_choice)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "98aaa12c",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3.8.13 ('nerfactory')",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.13"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "e691f033e0f2f1b9c0a11e7e81375a1e27aec87a71fd1b1eaa545f7a5e61b3f6"
+ }
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/nerfactory/cameras/cameras.py b/nerfactory/cameras/cameras.py
index 10e0b9a25..d46e3ba78 100644
--- a/nerfactory/cameras/cameras.py
+++ b/nerfactory/cameras/cameras.py
@@ -17,14 +17,16 @@ Camera Models
"""
import base64
from abc import abstractmethod
-from typing import Dict, Optional, Type
+from typing import Dict, Optional, Tuple, Type
import cv2
+import numpy as np
import torch
from torch.nn.functional import normalize
from torchtyping import TensorType
from nerfactory.cameras.rays import RayBundle
+from nerfactory.utils.misc import is_not_none
class Camera:
@@ -161,11 +163,15 @@ class Camera:
return ray_bundle
@abstractmethod
- def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict:
+ def to_json(
+ self,
+ image: Optional[TensorType["image_height", "image_width", 2]] = None,
+ resize_shape: Optional[Tuple[int, int]] = None,
+ ) -> Dict:
"""Converts the camera to a json dictionary.
Args:
- image: An image that is encoded to a base64 string. Defaults to None.
+ image: An image in range [0, 1] that is encoded to a base64 string. Defaults to None.
Returns:
A JSON representation of the camera
@@ -295,7 +301,11 @@ class PinholeCamera(Camera):
return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None])
- def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict:
+ def to_json(
+ self,
+ image: Optional[TensorType["image_height", "image_width", 2]] = None,
+ resize_shape: Optional[Tuple[int, int]] = None,
+ ) -> Dict:
json_ = {
"type": "PinholeCamera",
"cx": self.cx,
@@ -305,10 +315,11 @@ class PinholeCamera(Camera):
"camera_to_world": self.camera_to_world.tolist(),
"camera_index": self.camera_index,
}
- if image:
- # move image to cpu if not already
- # TODO: move to numpy
- data = cv2.imencode(".png", image[:, :, ::-1])[1].tobytes()
+ if is_not_none(image):
+ image_uint8 = (image * 255).detach().cpu().numpy().astype(np.uint8)
+ if resize_shape:
+ image_uint8 = cv2.resize(image_uint8, resize_shape)
+ data = cv2.imencode(".png", image_uint8)[1].tobytes()
json_["image"] = str("data:image/png;base64," + base64.b64encode(data).decode("ascii"))
return json_
@@ -346,9 +357,6 @@ class SimplePinholeCamera(PinholeCamera):
def fy_index(cls):
return 2
- def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict:
- raise NotImplementedError
-
@staticmethod
def from_json(json_: Dict) -> "SimplePinholeCamera": # pylint:disable=no-self-use
raise NotImplementedError
@@ -441,7 +449,11 @@ class EquirectangularCamera(Camera):
pixel_area = dx * dy
return RayBundle(origins=origins, directions=directions, pixel_area=pixel_area[..., None])
- def to_json(self, image: Optional[TensorType["image_height", "image_width", 2]] = None) -> Dict:
+ def to_json(
+ self,
+ image: Optional[TensorType["image_height", "image_width", 2]] = None,
+ resize_shape: Optional[Tuple[int, int]] = None,
+ ) -> Dict:
raise NotImplementedError
@staticmethod
diff --git a/nerfactory/data/image_dataset.py b/nerfactory/data/image_dataset.py
index da2792229..deb511fb7 100644
--- a/nerfactory/data/image_dataset.py
+++ b/nerfactory/data/image_dataset.py
@@ -87,7 +87,7 @@ class ImageDataset(Dataset):
assert image.shape[2] in [3, 4], f"Image shape of {image.shape} is in correct."
return image
- def get_image(self, image_idx: int):
+ def get_image(self, image_idx: int) -> TensorType["image_height", "image_width", "num_channels"]:
"""Returns a 3 channel image."""
image = torch.from_numpy(self.get_numpy_image(image_idx).astype("float32") / 255.0)
if self.alpha_color is not None and image.shape[-1] == 4:
diff --git a/nerfactory/graphs/base.py b/nerfactory/graphs/base.py
index e9512dd68..6c497c455 100644
--- a/nerfactory/graphs/base.py
+++ b/nerfactory/graphs/base.py
@@ -86,9 +86,6 @@ class Graph(nn.Module):
self.populate_fields()
self.populate_misc_modules() # populate the modules
self.callbacks = None
- # variable for visualizer to fetch TODO(figure out if there is cleaner way to do this)
- self.vis_outputs = None
- self.default_output_name = None
# to keep track of which device the nn.Module is on
self.device_indicator_param = nn.Parameter(torch.empty(0))
diff --git a/nerfactory/viewer/app/.env.development b/nerfactory/viewer/app/.env.development
index fd2617ea9..358cb9e29 100644
--- a/nerfactory/viewer/app/.env.development
+++ b/nerfactory/viewer/app/.env.development
@@ -1,4 +1,5 @@
BROWSER=none
FAST_REFRESH=false
HOST=localhost
-PORT=4000
\ No newline at end of file
+PORT=4000
+ESLINT_NO_DEV_ERRORS=true
\ No newline at end of file
diff --git a/nerfactory/viewer/app/.eslintrc.json b/nerfactory/viewer/app/.eslintrc.json
new file mode 100644
index 000000000..8c21d1348
--- /dev/null
+++ b/nerfactory/viewer/app/.eslintrc.json
@@ -0,0 +1,33 @@
+{
+ "env": {
+ "browser": true,
+ "es2021": true
+ },
+ "extends": [
+ "eslint:recommended",
+ "plugin:react/recommended",
+ "airbnb",
+ "prettier"
+ ],
+ "parser": "@typescript-eslint/parser",
+ "parserOptions": {
+ "ecmaFeatures": {
+ "jsx": true
+ },
+ "ecmaVersion": "latest",
+ "sourceType": "module"
+ },
+ "plugins": [
+ "react",
+ "@typescript-eslint"
+ ],
+ "rules": {
+ "arrow-body-style": "off",
+ "camelcase": "off",
+ "import/prefer-default-export": "off",
+ "no-alert": "off",
+ "no-console": "off",
+ "prefer-destructuring": "off",
+ "react/destructuring-assignment": "off"
+ }
+}
\ No newline at end of file
diff --git a/nerfactory/viewer/app/package.json b/nerfactory/viewer/app/package.json
index ad5827f0a..03739ef83 100644
--- a/nerfactory/viewer/app/package.json
+++ b/nerfactory/viewer/app/package.json
@@ -4,28 +4,37 @@
"version": "0.0.0",
"private": true,
"dependencies": {
+ "@reduxjs/toolkit": "^1.8.3",
"@testing-library/jest-dom": "^5.16.4",
"@testing-library/react": "^13.3.0",
"@testing-library/user-event": "^14.2.0",
"ccapture.js": "github:aceslowman/ccapture.js#npm-fix",
+ "classnames": "^2.3.1",
"dat.gui": "^0.7.9",
+ "eslint-config-prettier": "^8.5.0",
"leva": "^0.9.29",
"msgpack-lite": "^0.1.26",
+ "prop-types": "^15.8.1",
"react": "^18.1.0",
"react-dom": "^18.1.0",
- "react-scripts": "^5.0.1",
+ "react-redux": "^8.0.2",
+ "redux": "^4.2.0",
"socket.io-client": "^4.5.1",
"three": "^0.142.0",
"three-wtm": "^1.0",
"websocket": "^1.0.34",
- "wwobjloader2": "^4.0"
+ "wwobjloader2": "^4.0",
+ "electron": "^19.0.4",
+ "electron-is-dev": "^2.0.0"
},
"scripts": {
"start": "react-scripts start",
"build": "react-scripts build",
"test": "react-scripts test",
"eject": "react-scripts eject",
- "electron": "electron ."
+ "electron": "electron .",
+ "lint": "eslint --ext .js,.jsx .",
+ "lint:fix": "eslint --fix --ext .js,.jsx ."
},
"eslintConfig": {
"extends": "react-app"
@@ -47,10 +56,20 @@
"license": "ISC",
"description": "",
"devDependencies": {
+ "@svgr/webpack": "^6.2.1",
+ "prettier": "2.7.1",
"concurrently": "^7.2.1",
- "electron": "^19.0.4",
- "electron-is-dev": "^2.0.0",
+ "eslint": "^8.2.0",
+ "eslint-config-airbnb": "19.0.4",
+ "eslint-plugin-import": "^2.25.3",
+ "eslint-plugin-jsx-a11y": "^6.5.1",
+ "eslint-plugin-react": "^7.28.0",
+ "eslint-plugin-react-hooks": "^4.3.0",
+ "react-scripts": "^5.0.1",
"typescript": "^4.7.3",
"wait-on": "^6.0.1"
+ },
+ "resolutions": {
+ "@svgr/webpack": "$@svgr/webpack"
}
}
diff --git a/nerfactory/viewer/app/public/favicon.ico b/nerfactory/viewer/app/public/favicon.ico
deleted file mode 100644
index bcd5dfd67..000000000
Binary files a/nerfactory/viewer/app/public/favicon.ico and /dev/null differ
diff --git a/nerfactory/viewer/app/public/favicon.png b/nerfactory/viewer/app/public/favicon.png
new file mode 100644
index 000000000..559199e18
Binary files /dev/null and b/nerfactory/viewer/app/public/favicon.png differ
diff --git a/nerfactory/viewer/app/public/index.html b/nerfactory/viewer/app/public/index.html
index 9c12b199a..b7227e088 100644
--- a/nerfactory/viewer/app/public/index.html
+++ b/nerfactory/viewer/app/public/index.html
@@ -2,7 +2,7 @@
-
+
- nerfactory Viewer
+ Nerfviewer
diff --git a/nerfactory/viewer/app/public/logo192.png b/nerfactory/viewer/app/public/logo192.png
deleted file mode 100644
index fc44b0a37..000000000
Binary files a/nerfactory/viewer/app/public/logo192.png and /dev/null differ
diff --git a/nerfactory/viewer/app/public/logo512.png b/nerfactory/viewer/app/public/logo512.png
deleted file mode 100644
index a4e47a654..000000000
Binary files a/nerfactory/viewer/app/public/logo512.png and /dev/null differ
diff --git a/nerfactory/viewer/app/public/manifest.json b/nerfactory/viewer/app/public/manifest.json
index 080d6c77a..f9051fe71 100644
--- a/nerfactory/viewer/app/public/manifest.json
+++ b/nerfactory/viewer/app/public/manifest.json
@@ -1,23 +1,6 @@
{
"short_name": "React App",
"name": "Create React App Sample",
- "icons": [
- {
- "src": "favicon.ico",
- "sizes": "64x64 32x32 24x24 16x16",
- "type": "image/x-icon"
- },
- {
- "src": "logo192.png",
- "type": "image/png",
- "sizes": "192x192"
- },
- {
- "src": "logo512.png",
- "type": "image/png",
- "sizes": "512x512"
- }
- ],
"start_url": ".",
"display": "standalone",
"theme_color": "#000000",
diff --git a/nerfactory/viewer/app/src/App.css b/nerfactory/viewer/app/src/App.css
new file mode 100644
index 000000000..ec2585e8c
--- /dev/null
+++ b/nerfactory/viewer/app/src/App.css
@@ -0,0 +1,13 @@
+body {
+ margin: 0;
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
+ 'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
+ sans-serif;
+ -webkit-font-smoothing: antialiased;
+ -moz-osx-font-smoothing: grayscale;
+}
+
+code {
+ font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
+ monospace;
+}
diff --git a/nerfactory/viewer/app/src/App.jsx b/nerfactory/viewer/app/src/App.jsx
new file mode 100644
index 000000000..8c59f9410
--- /dev/null
+++ b/nerfactory/viewer/app/src/App.jsx
@@ -0,0 +1,18 @@
+import './App.css';
+
+import React from 'react';
+import { RenderControls } from './modules/ConfigPanel/ConfigPanel';
+
+import SetupScene from './modules/Scene/Scene';
+import ViewerWindow from './modules/ViewerWindow/ViewerWindow';
+
+export default function App() {
+ const scene = SetupScene();
+
+ return (
+
+
+
+
+ );
+}
diff --git a/nerfactory/viewer/app/src/ExtensibleObjectLoader.js b/nerfactory/viewer/app/src/ExtensibleObjectLoader.js
deleted file mode 100644
index 01346a452..000000000
--- a/nerfactory/viewer/app/src/ExtensibleObjectLoader.js
+++ /dev/null
@@ -1,282 +0,0 @@
-import * as THREE from "three";
-import { OBJLoader } from "three/examples/jsm/loaders/OBJLoader.js";
-import { ColladaLoader } from "three/examples/jsm/loaders/ColladaLoader.js";
-import { MTLLoader } from "three/examples/jsm/loaders/MTLLoader.js";
-import { MtlObjBridge } from "wwobjloader2";
-import { STLLoader } from "three/examples/jsm/loaders/STLLoader.js";
-import { PLYLoader } from "three/examples/jsm/loaders/PLYLoader.js";
-import * as BufferGeometryUtils from "three/examples/jsm/utils/BufferGeometryUtils.js";
-
-// Merges a hierarchy of collada mesh geometries into a single
-// `BufferGeometry` object:
-// * A new merged `BufferGeometry` if the input contains meshes
-// * empty `BufferGeometry` otherwise
-function merge_geometries(object, preserve_materials = false) {
- let materials = [];
- let geometries = [];
- let root_transform = object.matrix.clone();
-
- function collectGeometries(node, parent_transform) {
- let transform = parent_transform.clone().multiply(node.matrix);
- if (node.type === "Mesh") {
- node.geometry.applyMatrix(transform);
- geometries.push(node.geometry);
- materials.push(node.material);
- }
- for (let child of node.children) {
- collectGeometries(child, transform);
- }
- }
-
- collectGeometries(object, root_transform);
- let result = null;
- if (geometries.length === 1) {
- result = geometries[0];
- if (preserve_materials) {
- result.material = materials[0];
- }
- } else if (geometries.length > 1) {
- result = BufferGeometryUtils.mergeBufferGeometries(geometries, true);
- if (preserve_materials) {
- result.material = materials;
- }
- } else {
- result = new THREE.BufferGeometry();
- }
- return result;
-}
-
-// Handler for special texture types that we want to support
-// in addition to whatever three.js supports. This function
-// takes a json object representing a single texture, and should
-// return either:
-// * A new `THREE.Texture` if that json represents a special texture
-// * `null` otherwise
-function handle_special_texture(json) {
- if (json.type === "_text") {
- let canvas = document.createElement("canvas");
- // canvas width and height should be in the power of 2; otherwise although
- // the page usually loads successfully, WebGL does complain/warn
- canvas.width = 256;
- canvas.height = 256;
- let ctx = canvas.getContext("2d");
- ctx.textAlign = "center";
- let font_size = json.font_size;
- // auto-resing the font_size to fit in the canvas
- ctx.font = font_size + "px " + json.font_face;
- while (ctx.measureText(json.text).width > canvas.width) {
- font_size--;
- ctx.font = font_size + "px " + json.font_face;
- }
- ctx.fillText(json.text, canvas.width / 2, canvas.height / 2);
- let canvas_texture = new THREE.CanvasTexture(canvas);
- canvas_texture.uuid = json.uuid;
- return canvas_texture;
- } else {
- return null;
- }
-}
-
-// Handler for special geometry types that we want to support
-// in addition to whatever three.js supports. This function
-// takes a json object representing a single geometry, and should
-// return either:
-// * A new `THREE.Mesh` if that json represents a special geometry
-// * `null` otherwise
-function handle_special_geometry(geom) {
- if (geom.type === "_meshfile") {
- console.warn(
- "_meshfile is deprecated. Please use _meshfile_geometry for geometries and _meshfile_object for objects with geometry and material"
- );
- geom.type = "_meshfile_geometry";
- }
- if (geom.type === "_meshfile_geometry") {
- if (geom.format === "obj") {
- let loader = new OBJLoader();
- let obj = loader.parse(geom.data + "\n");
- let loaded_geom = merge_geometries(obj);
- loaded_geom.uuid = geom.uuid;
- return loaded_geom;
- } else if (geom.format === "dae") {
- let loader = new ColladaLoader();
- let obj = loader.parse(geom.data);
- let result = merge_geometries(obj.scene);
- result.uuid = geom.uuid;
- return result;
- } else if (geom.format === "stl") {
- let loader = new STLLoader();
- let loaded_geom = loader.parse(geom.data.buffer);
- loaded_geom.uuid = geom.uuid;
- return loaded_geom;
- } else if (geom.format === "ply") {
- let loader = new PLYLoader();
- let loaded_geom = loader.parse(geom.data.buffer);
- loaded_geom.uuid = geom.uuid;
- return loaded_geom;
- } else {
- console.error("Unsupported mesh type:", geom);
- return null;
- }
- }
- return null;
-}
-
-// The ExtensibleObjectLoader extends the THREE.ObjectLoader
-// interface, while providing some hooks for us to perform some
-// custom loading for things other than three.js native JSON.
-//
-// We currently use this class to support some extensions to
-// three.js JSON for objects which are easy to construct in
-// javascript but hard to construct in Python and/or Julia.
-// For example, we perform the following transformations:
-//
-// * Converting "_meshfile" geometries into actual meshes
-// using the THREE.js native mesh loaders
-// * Converting "_text" textures into text by drawing the
-// requested text onto a canvas.
-export class ExtensibleObjectLoader extends THREE.ObjectLoader {
- delegate(special_handler, base_handler, json, additional_objects) {
- let result = {};
- if (json === undefined) {
- return result;
- }
- let remaining_json = [];
- for (let data of json) {
- let x = special_handler(data);
- if (x !== null) {
- result[x.uuid] = x;
- } else {
- remaining_json.push(data);
- }
- }
- return Object.assign(
- result,
- base_handler(remaining_json, additional_objects)
- );
- }
-
- parseTextures(json, images) {
- return this.delegate(
- handle_special_texture,
- super.parseTextures,
- json,
- images
- );
- }
-
- parseGeometries(json, shapes) {
- return this.delegate(
- handle_special_geometry,
- super.parseGeometries,
- json,
- shapes
- );
- }
-
- parseObject(json, geometries, materials) {
- if (json.type === "_meshfile_object") {
- let geometry;
- let material;
- let manager = new THREE.LoadingManager();
- let path =
- json.url === undefined
- ? undefined
- : THREE.LoaderUtils.extractUrlBase(json.url);
- manager.setURLModifier((url) => {
- if (json.resources[url] !== undefined) {
- return json.resources[url];
- }
- return url;
- });
- if (json.format === "obj") {
- let loader = new OBJLoader(manager);
- if (json.mtl_library) {
- let mtl_loader = new MTLLoader(manager);
- let mtl_parse_result = mtl_loader.parse(json.mtl_library + "\n", "");
- let materials =
- MtlObjBridge.addMaterialsFromMtlLoader(mtl_parse_result);
- loader.setMaterials(materials);
- this.onTextureLoad();
- }
- let obj = loader.parse(json.data + "\n", path);
- geometry = merge_geometries(obj, true);
- geometry.uuid = json.uuid;
- material = geometry.material;
- } else if (json.format === "dae") {
- let loader = new ColladaLoader(manager);
- loader.onTextureLoad = this.onTextureLoad;
- let obj = loader.parse(json.data, path);
- geometry = merge_geometries(obj.scene, true);
- geometry.uuid = json.uuid;
- material = geometry.material;
- } else if (json.format === "stl") {
- let loader = new STLLoader();
- geometry = loader.parse(json.data.buffer, path);
- geometry.uuid = json.uuid;
- material = geometry.material;
- } else {
- console.error("Unsupported mesh type:", json);
- return null;
- }
- let object = new THREE.Mesh(geometry, material);
-
- // Copied from ObjectLoader
- object.uuid = json.uuid;
-
- if (json.name !== undefined) object.name = json.name;
-
- if (json.matrix !== undefined) {
- object.matrix.fromArray(json.matrix);
-
- if (json.matrixAutoUpdate !== undefined)
- object.matrixAutoUpdate = json.matrixAutoUpdate;
- if (object.matrixAutoUpdate)
- object.matrix.decompose(
- object.position,
- object.quaternion,
- object.scale
- );
- } else {
- if (json.position !== undefined)
- object.position.fromArray(json.position);
- if (json.rotation !== undefined)
- object.rotation.fromArray(json.rotation);
- if (json.quaternion !== undefined)
- object.quaternion.fromArray(json.quaternion);
- if (json.scale !== undefined) object.scale.fromArray(json.scale);
- }
-
- if (json.castShadow !== undefined) object.castShadow = json.castShadow;
- if (json.receiveShadow !== undefined)
- object.receiveShadow = json.receiveShadow;
-
- if (json.shadow) {
- if (json.shadow.bias !== undefined)
- object.shadow.bias = json.shadow.bias;
- if (json.shadow.radius !== undefined)
- object.shadow.radius = json.shadow.radius;
- if (json.shadow.mapSize !== undefined)
- object.shadow.mapSize.fromArray(json.shadow.mapSize);
- if (json.shadow.camera !== undefined)
- object.shadow.camera = this.parseObject(json.shadow.camera);
- }
-
- if (json.visible !== undefined) object.visible = json.visible;
- if (json.frustumCulled !== undefined)
- object.frustumCulled = json.frustumCulled;
- if (json.renderOrder !== undefined) object.renderOrder = json.renderOrder;
- if (json.userjson !== undefined) object.userjson = json.userData;
- if (json.layers !== undefined) object.layers.mask = json.layers;
-
- return object;
- } else if (json.type === "CameraHelper") {
- console.log("processing CameraHelper");
- console.log(json);
- console.log(geometries);
- console.log(materials);
- return super.parseObject(json, geometries, materials);
- } else {
- return super.parseObject(json, geometries, materials);
- }
- }
-}
diff --git a/nerfactory/viewer/app/src/SceneNode.js b/nerfactory/viewer/app/src/SceneNode.js
index 293def860..edfa23d58 100644
--- a/nerfactory/viewer/app/src/SceneNode.js
+++ b/nerfactory/viewer/app/src/SceneNode.js
@@ -1,12 +1,14 @@
-import * as THREE from "three";
+/* eslint-disable no-underscore-dangle */
+/* eslint-disable no-restricted-syntax */
+import * as THREE from 'three';
-const dat = require("dat.gui");
+const dat = require('dat.gui');
function remove_folders(gui) {
- for (let name of Object.keys(gui.__folders)) {
- let folder = gui.__folders[name];
+ for (const name of Object.keys(gui.__folders)) {
+ const folder = gui.__folders[name];
remove_folders(folder);
- dat.dom.dom.unbind(window, "resize", folder.__resizeHandler);
+ dat.dom.dom.unbind(window, 'resize', folder.__resizeHandler);
gui.removeFolder(folder);
}
}
@@ -20,7 +22,7 @@ function dispose(object) {
}
if (object.material) {
if (Array.isArray(object.material)) {
- for (let material of object.material) {
+ for (const material of object.material) {
if (material.map) {
material.map.dispose();
}
@@ -35,42 +37,41 @@ function dispose(object) {
}
}
-export class SceneNode {
+export default class SceneNode {
constructor(object, folder) {
this.object = object;
this.folder = folder;
this.children = {};
this.create_controls();
- for (let c of this.object.children) {
+ for (const c of this.object.children) {
this.add_child(c);
}
}
add_child(object) {
- let f = this.folder.addFolder(object.name);
- let node = new SceneNode(object, f);
+ const f = this.folder.addFolder(object.name);
+ const node = new SceneNode(object, f);
this.children[object.name] = node;
return node;
}
create_child(name) {
- let obj = new THREE.Group();
+ const obj = new THREE.Group();
obj.name = name;
this.object.add(obj);
return this.add_child(obj);
}
find(path) {
- if (path.length == 0) {
+ if (path.length === 0) {
return this;
- } else {
- let name = path[0];
- let child = this.children[name];
- if (child === undefined) {
- child = this.create_child(name);
- }
- return child.find(path.slice(1));
}
+ const name = path[0];
+ let child = this.children[name];
+ if (child === undefined) {
+ child = this.create_child(name);
+ }
+ return child.find(path.slice(1));
}
create_controls() {
@@ -79,19 +80,19 @@ export class SceneNode {
}
this.vis_controller = new dat.controllers.BooleanController(
this.object,
- "visible"
+ 'visible',
);
this.folder.domElement.prepend(this.vis_controller.domElement);
- this.vis_controller.domElement.style.height = "0";
- this.vis_controller.domElement.style.float = "right";
+ this.vis_controller.domElement.style.height = '0';
+ this.vis_controller.domElement.style.float = 'right';
}
set_property(property, value) {
- if (property === "position") {
+ if (property === 'position') {
this.object.position.set(value[0], value[1], value[2]);
- } else if (property === "quaternion") {
+ } else if (property === 'quaternion') {
this.object.quaternion.set(value[0], value[1], value[2], value[3]);
- } else if (property === "scale") {
+ } else if (property === 'scale') {
this.object.scale.set(value[0], value[1], value[2]);
} else {
this.object[property] = value;
@@ -100,17 +101,17 @@ export class SceneNode {
}
set_transform(matrix) {
- let mat = new THREE.Matrix4();
+ const mat = new THREE.Matrix4();
mat.fromArray(matrix);
mat.decompose(
this.object.position,
this.object.quaternion,
- this.object.scale
+ this.object.scale,
);
}
set_object(object) {
- let parent = this.object.parent;
+ const parent = this.object.parent;
this.dispose_recursive();
this.object.parent.remove(this.object);
this.object = object;
@@ -119,19 +120,19 @@ export class SceneNode {
}
dispose_recursive() {
- for (let name of Object.keys(this.children)) {
+ for (const name of Object.keys(this.children)) {
this.children[name].dispose_recursive();
}
dispose(this.object);
}
delete(path) {
- if (path.length == 0) {
+ if (path.length === 0) {
console.error("Can't delete an empty path");
} else {
- let parent = this.find(path.slice(0, path.length - 1));
- let name = path[path.length - 1];
- let child = parent.children[name];
+ const parent = this.find(path.slice(0, path.length - 1));
+ const name = path[path.length - 1];
+ const child = parent.children[name];
if (child !== undefined) {
child.dispose_recursive();
parent.object.remove(child.object);
diff --git a/nerfactory/viewer/app/src/Viewer.jsx b/nerfactory/viewer/app/src/Viewer.jsx
deleted file mode 100644
index 0e25cfbee..000000000
--- a/nerfactory/viewer/app/src/Viewer.jsx
+++ /dev/null
@@ -1,461 +0,0 @@
-import React, { Component } from "react";
-import * as THREE from "three";
-import { OrbitControls } from "three/examples/jsm/controls/OrbitControls";
-import Stats from "three/examples/jsm/libs/stats.module.js";
-import { GUI } from "dat.gui";
-import { split_path } from "./utils";
-import { SceneNode } from "./SceneNode";
-import { ExtensibleObjectLoader } from "./ExtensibleObjectLoader";
-
-function websocket_endpoint_from_url(url) {
- let endpoint = url.split("?").pop();
- if (endpoint == "") {
- let message =
- "Please set the websocket endpoint. E.g., a correct URL may be: http://localhost:4000?localhost:8051";
- window.alert(message);
- return null;
- }
- return endpoint;
-}
-
-function findCameraObjectUnderObject3D(object) {
- // seach the tree for the camera
- if (object instanceof THREE.Camera) {
- return object;
- }
- for (let name of Object.keys(object.children)) {
- return findCameraObjectUnderObject3D(object.children[name]);
- }
- return null;
-}
-
-var msgpack = require("msgpack-lite");
-
-export class ViewerState extends Component {
- constructor(props) {
- super(props);
- this.state = {
- scene: null,
- controls: null,
- renderer: null,
- gui: null,
- scene_tree: null,
- viewport_width: null,
- viewport_height: null,
- };
- this.update = this.update.bind(this);
- this.set_object = this.set_object.bind(this);
- this.handle_command = this.handle_command.bind(this);
- this.threejs_ref = React.createRef();
- this.props = props;
- }
-
- handleResize() {
- this.state.viewport_width = this.getViewportWidth();
- this.state.viewport_height = this.getViewportHeight();
- this.state.camera_main.aspect =
- this.state.viewport_width / this.state.viewport_height;
- this.state.camera_main.updateProjectionMatrix();
- this.state.renderer_main.setSize(
- this.state.viewport_width,
- this.state.viewport_height
- );
- }
-
- update() {
- requestAnimationFrame(this.update);
- this.handleResize();
- let camera_main = findCameraObjectUnderObject3D(
- this.state.scene_tree.find(["Cameras", "Main Camera"]).object
- );
- camera_main.updateProjectionMatrix();
- if (this.state.controls_main != null) {
- this.state.controls_main.update();
- }
- this.state.renderer_main.render(this.state.scene, camera_main);
- this.send_camera_over_websocket();
- }
-
- send_camera_over_websocket() {
- /* update the camera information in the python server
- if the websocket is connected */
- // console.log("send_camera_over_websocket");
- if (this.state.websocket.readyState === WebSocket.OPEN) {
- // update the camera information in the python server
- let camera_main = findCameraObjectUnderObject3D(
- this.state.scene_tree.find(["Cameras", "Main Camera"]).object
- );
- // console.log(camera_main.toJSON());
- let cmd = "set_object";
- let path = "Cameras/Main Camera";
- let data = {
- type: cmd,
- path: path,
- object: camera_main.toJSON(),
- };
- let message = msgpack.encode(data);
- this.state.websocket.send(message);
- }
- }
-
- send_training_state_over_websocket(value) {
- if (this.state.websocket.readyState === WebSocket.OPEN) {
- let cmd = "set_training_state";
- let path = "Training State";
- let data = {
- type: cmd,
- path: path,
- training_state: value,
- };
- let message = msgpack.encode(data);
- this.state.websocket.send(message);
- }
- }
-
- send_output_type_over_websocket(value) {
- /* update the output option in the python server
- if the user changes selection */
- if (this.state.websocket.readyState === WebSocket.OPEN) {
- let cmd = "set_output_type";
- let path = "Output Type";
- let data = {
- type: cmd,
- path: path,
- output_type: value,
- };
- let message = msgpack.encode(data);
- this.state.websocket.send(message);
- }
- }
-
- send_min_resolution_over_websocket(value) {
- if (this.state.websocket.readyState === WebSocket.OPEN) {
- let cmd = "set_min_resolution";
- let path = "Min Resolution";
- let data = {
- type: cmd,
- path: path,
- min_resolution: value,
- };
- let message = msgpack.encode(data);
- this.state.websocket.send(message);
- }
- }
-
- send_max_resolution_over_websocket(value) {
- if (this.state.websocket.readyState === WebSocket.OPEN) {
- let cmd = "set_max_resolution";
- let path = "Max Resolution";
- let data = {
- type: cmd,
- path: path,
- max_resolution: value,
- };
- let message = msgpack.encode(data);
- this.state.websocket.send(message);
- }
- }
-
- set_object(path, object) {
- if (!(object instanceof THREE.Camera)) {
- this.state.scene_tree.find(path.concat(["