mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-17 13:23:46 +08:00
Integrating WebRTC into viewer (#73)
* restructuring for webrtc integration * some webrtc code but not working * webrtc connects when muted * the flag is real! * updating camera in websocket bridge. moved webrtc to existing websocket connection instead of http request. need to change video track * rudimentary adding real-time imaging * added free TURN server so code works everywhere * slow rendering works during training with the viewer * fix tensor device and update default model save dir * forgot to include trainer updates * removing some code that isn't needed * quick fix to bins. not the most accurate however * cleaned up code and added some config logic * black * adding license Co-authored-by: evonneng <evonne_ng@berkeley.edu>
This commit is contained in:
1 parent
e507c97e3e
commit
bf13ed3cf3
28 files changed
+8923
-917
No files matched your search
@@ -98,3 +98,10 @@ optimizers:
|
||||
_target_: pyrad.optimizers.optimizers.ExponentialDecaySchedule
|
||||
lr_final: 0.000005
|
||||
max_steps: 1000000
|
||||
|
||||
viewer:
|
||||
enable: False
|
||||
zmq_url: tcp://127.0.0.1:6000
|
||||
render_image_height: 100
|
||||
steps_per_render_image: 100
|
||||
num_rays_per_chunk: 4096
|
||||
@@ -23,3 +23,9 @@ optimizers:
|
||||
optimizer:
|
||||
lr: 1e-2
|
||||
scheduler: null
|
||||
|
||||
viewer:
|
||||
enable: True
|
||||
render_image_height: 200
|
||||
steps_per_render_image: 5
|
||||
num_rays_per_chunk: 16384
|
||||
@@ -1,3 +1,6 @@
|
||||
aiohttp
|
||||
aiortc
|
||||
av
|
||||
black[jupyter]==22.3.0
|
||||
functorch==0.1.1
|
||||
h5py>=2.9.0
|
||||
|
||||
@@ -10,7 +10,20 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"execution_count": null,
|
||||
"id": "8501e01d-f932-4a75-8259-2def67a546d7",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
|
||||
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b15dfdc1-5054-40bc-9631-4c7426eecfd9",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -35,6 +48,8 @@
|
||||
"from pyrad.utils.plotly import get_line_segments_from_lines\n",
|
||||
"from pyrad.cameras.cameras import get_camera_model\n",
|
||||
"from pyrad.utils.misc import get_dict_to_torch, instantiate_from_dict_config\n",
|
||||
"from pyrad.engine.trainer import Trainer\n",
|
||||
"from pyrad.utils.config import Config, setup_config\n",
|
||||
"\n",
|
||||
"# the new import\n",
|
||||
"from pyrad.viewer.backend import vis_utils\n",
|
||||
@@ -44,19 +59,31 @@
|
||||
"from omegaconf import open_dict\n",
|
||||
"import pprint\n",
|
||||
"from tqdm import tqdm\n",
|
||||
"import random"
|
||||
"import random\n",
|
||||
"\n",
|
||||
"import umsgpack\n",
|
||||
"from pyrad.viewer.backend.utils import get_chunks, get_intrinsics_matrix_and_camera_to_world_h"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"id": "df82e083-e261-43e0-a1dc-05c6a59d16e1",
|
||||
"execution_count": null,
|
||||
"id": "032e4c9e-25be-4cc0-9ac8-a5be723b775a",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# establish the connection to the tcp backend, to talk to the visualizer\n",
|
||||
"# zmq_url should match the output of `python run_zmq_server.py`\n",
|
||||
"vis = vis_utils.get_vis(zmq_url=\"tcp://127.0.0.1:6000\")\n",
|
||||
"vis = vis_utils.get_vis(zmq_url=\"tcp://127.0.0.1:6000\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "df82e083-e261-43e0-a1dc-05c6a59d16e1",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"vis.delete()\n",
|
||||
"\n",
|
||||
"# draws a red box in the scene\n",
|
||||
@@ -65,18 +92,99 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"execution_count": null,
|
||||
"id": "7fe7478d",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"with initialize(version_base=None, config_path=\"../configs\"):\n",
|
||||
" config = compose(config_name=\"graph_default.yaml\")"
|
||||
" config = compose(\n",
|
||||
" config_name=\"graph_default.yaml\",\n",
|
||||
" overrides=[\n",
|
||||
" \"trainer.resume_train.load_dir=/shared/ethanweber/pyrad/outputs/blender_lego/vanilla_nerf/2022-07-04_221907/checkpoints\",\n",
|
||||
" \"trainer.resume_train.load_step=4000\",\n",
|
||||
" ],\n",
|
||||
" )\n",
|
||||
" config = setup_config(config)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"execution_count": null,
|
||||
"id": "5d231e36-dd38-489b-a3a3-860cdf49e551",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trainer = Trainer(config, local_rank=0, world_size=1)\n",
|
||||
"trainer.setup()\n",
|
||||
"trainer.draw_scene_in_viewer()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "1aba9ecb-0022-4896-833d-577abe532d23",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"aabb = trainer.dataset_inputs_train.scene_bounds.aabb\n",
|
||||
"vis_utils.draw_aabb(trainer.vis, aabb, name=\"dataset_inputs_train/scene_bounds/aabb\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "4b23fc89-bfc3-40c4-abba-fad1317aeb21",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5a829d31-4fc6-4908-963d-7326d3aa52da",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"outputs = trainer.render_image_in_viewer()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "84bd64de-2e0c-458e-a4b7-1084f1cca1c5",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"trainer.vis.delete()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "6ed9cdfa-6359-4a70-95c7-033e853e9b50",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"while True:\n",
|
||||
" outputs = trainer.render_image_in_viewer()\n",
|
||||
" image = outputs[\"rgb_fine\"].cpu().numpy().astype(\"float64\") * 255\n",
|
||||
" trainer.vis[\"/Cameras/Main Camera\"].set_image(image)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "b3bcd221-05a8-494e-b817-3e2de7f5a0df",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"image.shape"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "642b0a25-0e88-4eac-b661-a57a74fb10fc",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -86,7 +194,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"execution_count": null,
|
||||
"id": "938f090b-3b28-4fd6-82dc-d4b5c5917134",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
@@ -97,45 +205,10 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"id": "5d014a6b-50fa-48ab-b135-0ab2b81c7529",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# grab 10 images and show their frustums\n",
|
||||
"indices = random.sample(range(len(image_dataset_train)), k=10)\n",
|
||||
"for idx in indices:\n",
|
||||
" image = image_dataset_train[idx][\"image\"]\n",
|
||||
" camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None)\n",
|
||||
" pose = camera.get_camera_to_world().double().numpy()\n",
|
||||
" K = camera.get_intrinsics_matrix().double().numpy()\n",
|
||||
" vis_utils.draw_camera_frustum(\n",
|
||||
" vis,\n",
|
||||
" image=(image.double().numpy() * 255.0),\n",
|
||||
" pose=pose,\n",
|
||||
" K=K,\n",
|
||||
" height=1.0,\n",
|
||||
" name=\"{:06d}\".format(idx),\n",
|
||||
" displayed_focal_length=0.5,\n",
|
||||
" realistic=False,\n",
|
||||
" )"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": null,
|
||||
"id": "fd04703a-aa50-45e4-95fd-4d881ff18d61",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"estimated_time: 3.3333333333333335\n",
|
||||
"time_elapsed: 3.5769083499908447\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# move a camera around\n",
|
||||
"num_cameras = len(dataset_inputs.intrinsics)\n",
|
||||
|
||||
+88
-8
@@ -17,13 +17,16 @@ Code to train model.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from typing import Dict
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import umsgpack
|
||||
from torch.nn.parallel import DistributedDataParallel as DDP
|
||||
from torchtyping import TensorType
|
||||
|
||||
from pyrad.cameras.cameras import get_camera, get_intrinsics_from_intrinsics_matrix
|
||||
from pyrad.cameras.rays import RayBundle
|
||||
from pyrad.data.dataloader import EvalDataloader, setup_dataset_eval, setup_dataset_train
|
||||
from pyrad.graphs.base import setup_graph
|
||||
@@ -32,6 +35,9 @@ from pyrad.utils import profiler, writer
|
||||
from pyrad.utils.config import Config
|
||||
from pyrad.utils.decorators import check_main_thread
|
||||
from pyrad.utils.writer import EventName, TimeWriter
|
||||
from pyrad.viewer.backend import vis_utils
|
||||
from pyrad.viewer.backend.utils import get_intrinsics_matrix_and_camera_to_world_h
|
||||
from pyrad.viewer.backend.visualizer import ViewerWindow, Visualizer
|
||||
|
||||
logging.getLogger("PIL").setLevel(logging.WARNING)
|
||||
|
||||
@@ -40,10 +46,9 @@ class Trainer:
|
||||
"""Training class
|
||||
|
||||
Args:
|
||||
config (Config): _description_
|
||||
local_rank (int, optional): _description_. Defaults to 0.
|
||||
world_size (int, optional): _description_. Defaults to 1.
|
||||
cpu (bool, optional): Whether or not to use the CPU.
|
||||
config (Config): The configuration object.
|
||||
local_rank (int, optional): Local rank of the process. Defaults to 0.
|
||||
world_size (int, optional): World size of the process. Defaults to 1.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Config, local_rank: int = 0, world_size: int = 1):
|
||||
@@ -52,6 +57,7 @@ class Trainer:
|
||||
self.world_size = world_size
|
||||
self.device = "cpu" if world_size == 0 else f"cuda:{local_rank}"
|
||||
# dataset variables
|
||||
self.dataset_inputs_train = None
|
||||
self.dataloader_train = None
|
||||
self.dataloader_eval = None
|
||||
# model variables
|
||||
@@ -62,11 +68,20 @@ class Trainer:
|
||||
writer.setup_event_writers(config.logging, max_iter=config.trainer.max_num_iterations)
|
||||
profiler.setup_profiler(config.logging)
|
||||
|
||||
self.vis = None
|
||||
if self.config.viewer.enable:
|
||||
window = ViewerWindow(zmq_url=self.config.viewer.zmq_url)
|
||||
self.vis = Visualizer(window=window)
|
||||
logging.info("Connected to viewer at %s", self.config.viewer.zmq_url)
|
||||
self.vis.delete()
|
||||
else:
|
||||
logging.info("Continuing without viewer.")
|
||||
|
||||
def setup(self, test_mode=False):
|
||||
"""Setup the Trainer by calling other setup functions."""
|
||||
dataset_inputs_train, self.dataloader_train = setup_dataset_train(self.config.data, device=self.device)
|
||||
self.dataset_inputs_train, self.dataloader_train = setup_dataset_train(self.config.data, device=self.device)
|
||||
_, self.dataloader_eval = setup_dataset_eval(self.config.data, test_mode=test_mode, device=self.device)
|
||||
self.graph = setup_graph(self.config.graph, dataset_inputs_train, device=self.device)
|
||||
self.graph = setup_graph(self.config.graph, self.dataset_inputs_train, device=self.device)
|
||||
self.optimizers = setup_optimizers(self.config.optimizers, self.graph.get_param_groups())
|
||||
|
||||
if self.config.trainer.resume_train.load_dir:
|
||||
@@ -90,7 +105,11 @@ class Trainer:
|
||||
return loss_sum
|
||||
|
||||
def train(self) -> None:
|
||||
"""_summary_"""
|
||||
"""Train the model."""
|
||||
|
||||
if self.vis:
|
||||
self.draw_scene_in_viewer()
|
||||
|
||||
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
|
||||
num_iterations = self.config.trainer.max_num_iterations
|
||||
iter_dataloader_train = iter(self.dataloader_train)
|
||||
@@ -106,6 +125,13 @@ class Trainer:
|
||||
writer.put_dict(name="Loss/train-loss_dict", scalar_dict=loss_dict, step=step)
|
||||
if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0:
|
||||
self._save_checkpoint(self.config.trainer.model_dir, step)
|
||||
if (
|
||||
self.vis
|
||||
and step != 0
|
||||
and self.config.viewer.steps_per_render_image
|
||||
and step % self.config.viewer.steps_per_render_image == 0
|
||||
):
|
||||
_ = self.render_image_in_viewer()
|
||||
if step % self.config.trainer.steps_per_test == 0:
|
||||
self.eval_with_dataloader(self.dataloader_eval, step=step)
|
||||
self._write_out_storage(step)
|
||||
@@ -128,7 +154,7 @@ class Trainer:
|
||||
|
||||
def _load_checkpoint(self) -> None:
|
||||
"""helper function to load graph and optimizer from prespecified checkpoint"""
|
||||
load_config = self.config.resume_train
|
||||
load_config = self.config.trainer.resume_train
|
||||
load_path = os.path.join(load_config.load_dir, f"step-{load_config.load_step:09d}.ckpt")
|
||||
assert os.path.exists(load_path), f"Checkpoint {load_path} does not exist"
|
||||
loaded_state = torch.load(load_path, map_location="cpu")
|
||||
@@ -200,6 +226,60 @@ class Trainer:
|
||||
self.graph.train()
|
||||
return psnr
|
||||
|
||||
def draw_scene_in_viewer(self):
|
||||
"""Draw some images and the scene aabb in the viewer."""
|
||||
image_dataset_train = self.dataloader_train.image_sampler.dataset
|
||||
dataset_inputs = self.dataset_inputs_train
|
||||
indices = random.sample(range(len(image_dataset_train)), k=10)
|
||||
for idx in indices:
|
||||
image = image_dataset_train[idx]["image"]
|
||||
camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None)
|
||||
pose = camera.get_camera_to_world().double().numpy()
|
||||
K = camera.get_intrinsics_matrix().double().numpy()
|
||||
vis_utils.draw_camera_frustum(
|
||||
self.vis,
|
||||
image=(image.double().numpy() * 255.0),
|
||||
pose=pose,
|
||||
K=K,
|
||||
height=1.0,
|
||||
name=f"image_dataset_train/{idx:06d}",
|
||||
displayed_focal_length=0.5,
|
||||
realistic=False,
|
||||
)
|
||||
aabb = dataset_inputs.scene_bounds.aabb
|
||||
vis_utils.draw_aabb(self.vis, aabb, name="dataset_inputs_train/scene_bounds/aabb")
|
||||
|
||||
@profiler.time_function
|
||||
def render_image_in_viewer(self):
|
||||
"""
|
||||
Draw an image using the current camera pose from the viewer.
|
||||
The image is sent of a TCP connection and then uses WebRTC to send it to the viewer.
|
||||
"""
|
||||
data = self.vis["/Cameras/Main Camera"].get_object()
|
||||
message = umsgpack.unpackb(data)
|
||||
camera_object = message["object"]["object"]
|
||||
image_height = self.config.viewer.render_image_height
|
||||
intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h(
|
||||
camera_object, image_height=image_height
|
||||
)
|
||||
|
||||
camera_to_world = camera_to_world_h[:3, :]
|
||||
intrinsics = get_intrinsics_from_intrinsics_matrix(intrinsics_matrix)
|
||||
camera = get_camera(intrinsics, camera_to_world)
|
||||
camera_ray_bundle = camera.get_camera_ray_bundle(device=self.device)
|
||||
camera_ray_bundle.num_rays_per_chunk = self.config.viewer.num_rays_per_chunk
|
||||
|
||||
self.graph.eval()
|
||||
outputs = self.graph.get_outputs_for_camera_ray_bundle(camera_ray_bundle)
|
||||
self.graph.train()
|
||||
|
||||
# gross hack to get the image key, depending on which keys the graph uses
|
||||
rgb_key = "rgb" if "rgb" in outputs else "rgb_fine"
|
||||
# TODO: make it such that the TCP connection doesn't need float64
|
||||
image = outputs[rgb_key].cpu().numpy().astype("float64") * 255
|
||||
self.vis["/Cameras/Main Camera"].set_image(image)
|
||||
return outputs
|
||||
|
||||
def eval_with_dataloader(self, dataloader: EvalDataloader, step: int = None) -> None:
|
||||
"""Run evaluation with a given dataloader.
|
||||
|
||||
|
||||
@@ -116,6 +116,7 @@ class Config:
|
||||
data: DataConfig = MISSING
|
||||
graph: GraphConfig = MISSING
|
||||
optimizers: Dict[str, Any] = MISSING
|
||||
viewer: Dict[str, Any] = MISSING
|
||||
# additional optional parameters here
|
||||
hydra: Optional[Dict[str, Any]] = None
|
||||
|
||||
@@ -137,6 +138,7 @@ def setup_config(config: DictConfig) -> Config:
|
||||
data = DataConfig(**config.data)
|
||||
graph = GraphConfig(**config.graph)
|
||||
optimizers = config.optimizers
|
||||
viewer = config.viewer
|
||||
return Config(
|
||||
machine=machine,
|
||||
logging=logging,
|
||||
@@ -146,4 +148,5 @@ def setup_config(config: DictConfig) -> Config:
|
||||
data=data,
|
||||
graph=graph,
|
||||
optimizers=optimizers,
|
||||
viewer=viewer,
|
||||
)
|
||||
@@ -1,14 +1,12 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta name="theme-color" content="#000000" />
|
||||
<meta
|
||||
name="description"
|
||||
content="Web site created using create-react-app"
|
||||
/>
|
||||
<meta name="description" content="Web site created using create-react-app" />
|
||||
<link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
|
||||
<!--
|
||||
manifest.json provides metadata used when your web app is installed on a
|
||||
@@ -24,9 +22,10 @@
|
||||
work correctly both with client-side routing and a non-root public URL.
|
||||
Learn how to configure a non-root public URL by running `npm run build`.
|
||||
-->
|
||||
<title>visualizer</title>
|
||||
</head>
|
||||
<body>
|
||||
<title>PyRad Viewer</title>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
<div id="root"></div>
|
||||
<!--
|
||||
@@ -39,5 +38,6 @@
|
||||
To begin the development, run `npm start` or `yarn start`.
|
||||
To create a production bundle, use `npm run build` or `yarn build`.
|
||||
-->
|
||||
</body>
|
||||
</html>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
@@ -1,38 +0,0 @@
|
||||
.App {
|
||||
/*text-align: center;*/
|
||||
}
|
||||
|
||||
.App-logo {
|
||||
height: 40vmin;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
@media (prefers-reduced-motion: no-preference) {
|
||||
.App-logo {
|
||||
animation: App-logo-spin infinite 20s linear;
|
||||
}
|
||||
}
|
||||
|
||||
.App-header {
|
||||
background-color: #282c34;
|
||||
min-height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
font-size: calc(10px + 2vmin);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.App-link {
|
||||
color: #61dafb;
|
||||
}
|
||||
|
||||
@keyframes App-logo-spin {
|
||||
from {
|
||||
transform: rotate(0deg);
|
||||
}
|
||||
to {
|
||||
transform: rotate(360deg);
|
||||
}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
import React, { Component } from "react";
|
||||
import "./App.css";
|
||||
import { Viewer } from "./Viewer";
|
||||
|
||||
class Image extends Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {};
|
||||
}
|
||||
}
|
||||
|
||||
class ColorGUIHelper {
|
||||
constructor(object, prop) {
|
||||
this.object = object;
|
||||
this.prop = prop;
|
||||
}
|
||||
|
||||
get value() {
|
||||
return `#${this.object[this.prop].getHexString()}`;
|
||||
}
|
||||
|
||||
set value(hexString) {
|
||||
this.object[this.prop].set(hexString);
|
||||
}
|
||||
}
|
||||
|
||||
class App extends Component {
|
||||
constructor(props) {
|
||||
super(props);
|
||||
this.state = {
|
||||
client: null,
|
||||
};
|
||||
this.viewer_ref = React.createRef();
|
||||
}
|
||||
|
||||
render() {
|
||||
return (
|
||||
<div className="App">
|
||||
<Viewer ref={this.viewer_ref} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export default App;
|
||||
@@ -1,9 +0,0 @@
|
||||
import React from 'react';
|
||||
import { render } from '@testing-library/react';
|
||||
import App from './App';
|
||||
|
||||
test('renders learn react link', () => {
|
||||
const { getByText } = render(<App />);
|
||||
const linkElement = getByText(/learn react/i);
|
||||
expect(linkElement).toBeInTheDocument();
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import React, { Component } from "react";
|
||||
|
||||
export class BackgroundDiv extends Component {
|
||||
componentDidMount() {
|
||||
document.getElementById("BackgroundDiv-video").play();
|
||||
}
|
||||
render() {
|
||||
return (
|
||||
<div className="BackgroundDiv">
|
||||
<video id="BackgroundDiv-video" autoPlay muted loop>
|
||||
<source
|
||||
src="https://www.w3schools.com/html/mov_bbb.mp4"
|
||||
type="video/mp4"
|
||||
></source>
|
||||
</video>{" "}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -36,11 +36,10 @@ function dispose(object) {
|
||||
}
|
||||
|
||||
export class SceneNode {
|
||||
constructor(object, folder, on_update) {
|
||||
constructor(object, folder) {
|
||||
this.object = object;
|
||||
this.folder = folder;
|
||||
this.children = {};
|
||||
this.on_update = on_update;
|
||||
this.create_controls();
|
||||
for (let c of this.object.children) {
|
||||
this.add_child(c);
|
||||
@@ -49,7 +48,7 @@ export class SceneNode {
|
||||
|
||||
add_child(object) {
|
||||
let f = this.folder.addFolder(object.name);
|
||||
let node = new SceneNode(object, f, this.on_update);
|
||||
let node = new SceneNode(object, f);
|
||||
this.children[object.name] = node;
|
||||
return node;
|
||||
}
|
||||
@@ -79,28 +78,9 @@ export class SceneNode {
|
||||
this.folder.domElement.removeChild(this.vis_controller.domElement);
|
||||
}
|
||||
this.vis_controller = new dat.controllers.BooleanController(this.object, "visible");
|
||||
this.vis_controller.onChange(() => this.on_update());
|
||||
this.folder.domElement.prepend(this.vis_controller.domElement);
|
||||
this.vis_controller.domElement.style.height = "0";
|
||||
this.vis_controller.domElement.style.float = "right";
|
||||
|
||||
// // add on hover controls and clickable functionality
|
||||
// // hover
|
||||
// this.folder.domElement.addEventListener("mouseenter", function (event) {
|
||||
// event.target.style.color = "purple";
|
||||
// }, false);
|
||||
// this.folder.domElement.addEventListener("mouseleave", function (event) {
|
||||
// event.target.style.color = "";
|
||||
// }, false);
|
||||
|
||||
// // click
|
||||
// this.folder.domElement.addEventListener("dblclick", () => this.printValue(), false);
|
||||
}
|
||||
|
||||
printValue() {
|
||||
console.log("calling print value");
|
||||
console.log(this.object);
|
||||
console.log(this);
|
||||
}
|
||||
|
||||
set_property(property, value) {
|
||||
|
||||
+165
-332
@@ -1,5 +1,4 @@
|
||||
import React, { Component } from "react";
|
||||
import "./App.css";
|
||||
import * as THREE from "three";
|
||||
import { TrackballControls } from "three/examples/jsm/controls/TrackballControls";
|
||||
import Stats from "three/examples/jsm/libs/stats.module.js";
|
||||
@@ -12,26 +11,17 @@ function websocket_endpoint_from_url(url) {
|
||||
let endpoint = url.split("/").pop();
|
||||
if (endpoint == "") {
|
||||
let message =
|
||||
"Please set a websocket endpoint. For example, a correct URL may look like the following: http://localhost:3000/visiongpu10.csail.mit.edu:8051";
|
||||
"Please set the websocket endpoint. E.g., a correct URL may be: http://localhost:4000/localhost:8051";
|
||||
window.alert(message);
|
||||
return null;
|
||||
}
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
function removeAllChildNodes(parent) {
|
||||
// https://www.javascripttutorial.net/dom/manipulating/remove-all-child-nodes/
|
||||
while (parent.firstChild) {
|
||||
parent.removeChild(parent.firstChild);
|
||||
}
|
||||
}
|
||||
|
||||
function findCameraObjectUnderObject3D(object) {
|
||||
// seach the tree for the camera
|
||||
if (object instanceof THREE.Camera) {
|
||||
return object;
|
||||
} else if (object instanceof THREE.CameraHelper) {
|
||||
return object.camera;
|
||||
}
|
||||
for (let name of Object.keys(object.children)) {
|
||||
return findCameraObjectUnderObject3D(object.children[name]);
|
||||
@@ -50,87 +40,51 @@ export class Viewer extends Component {
|
||||
renderer: null,
|
||||
gui: null,
|
||||
scene_tree: null,
|
||||
needs_render: true,
|
||||
viewport_width: null,
|
||||
viewport_height: null,
|
||||
images: [],
|
||||
image_index: 0,
|
||||
};
|
||||
this.animate = this.animate.bind(this);
|
||||
this.update = this.update.bind(this);
|
||||
this.set_object = this.set_object.bind(this);
|
||||
this.handle_command = this.handle_command.bind(this);
|
||||
|
||||
// window.addEventListener('resize', this.handleResize.bind(this));
|
||||
}
|
||||
|
||||
set_dirty() {
|
||||
this.setState({ needs_render: true });
|
||||
this.threejs_ref = React.createRef();
|
||||
}
|
||||
|
||||
handleResize() {
|
||||
this.state.viewport_width = this.getViewportWidth();
|
||||
this.state.viewport_height = this.getViewportHeight();
|
||||
this.state.camera_l.aspect =
|
||||
this.state.camera_main.aspect =
|
||||
this.state.viewport_width / this.state.viewport_height;
|
||||
this.state.camera_l.updateProjectionMatrix();
|
||||
this.state.camera_r.updateProjectionMatrix();
|
||||
this.state.camera_orth.updateProjectionMatrix();
|
||||
this.state.renderer_l.setSize(
|
||||
this.state.camera_main.updateProjectionMatrix();
|
||||
this.state.renderer_main.setSize(
|
||||
this.state.viewport_width,
|
||||
(this.state.viewport_height * 2) / 3
|
||||
);
|
||||
this.state.renderer_r.setSize(
|
||||
this.state.viewport_width / 2,
|
||||
(this.state.viewport_height * 1) / 3
|
||||
);
|
||||
this.state.renderer_orth.setSize(
|
||||
this.state.viewport_width / 2,
|
||||
(this.state.viewport_height * 1) / 3
|
||||
this.state.viewport_height
|
||||
);
|
||||
}
|
||||
|
||||
animate() {
|
||||
requestAnimationFrame(this.animate);
|
||||
if (this.state.needs_render) {
|
||||
this.handleResize();
|
||||
let camera_l = findCameraObjectUnderObject3D(
|
||||
this.state.scene_tree.find(["Cameras", "Main Camera L"]).object
|
||||
);
|
||||
let camera_r = findCameraObjectUnderObject3D(
|
||||
this.state.scene_tree.find(["Cameras", "Main Camera R"]).object
|
||||
);
|
||||
let camera_orth = findCameraObjectUnderObject3D(
|
||||
this.state.scene_tree.find(["Cameras", "Main Camera Orth"]).object
|
||||
);
|
||||
camera_l.updateProjectionMatrix();
|
||||
camera_r.updateProjectionMatrix();
|
||||
camera_orth.updateProjectionMatrix();
|
||||
if (this.state.controls_l != null) {
|
||||
this.state.controls_l.update();
|
||||
}
|
||||
if (this.state.controls_r != null) {
|
||||
this.state.controls_r.update();
|
||||
}
|
||||
if (this.state.controls_orth != null) {
|
||||
this.state.controls_orth.update();
|
||||
}
|
||||
this.state.renderer_l.render(this.state.scene, camera_l);
|
||||
this.state.renderer_r.render(this.state.scene, camera_r);
|
||||
|
||||
// NOTE(ethan): trying to add background image
|
||||
|
||||
// console.log("ethan is here");
|
||||
if (this.state.websocket_is_opened) {
|
||||
this.state.websocket.send("TODO: put camera pose here");
|
||||
}
|
||||
|
||||
this.state.renderer_orth.render(
|
||||
this.backgroundScene,
|
||||
this.backgroundCamera
|
||||
);
|
||||
// this.state.renderer_orth.render(this.state.scene, camera_orth);
|
||||
this.state.needs_render = false;
|
||||
this.set_dirty(); // TODO(ethan): remove this but make sure to include it everywhere
|
||||
update() {
|
||||
requestAnimationFrame(this.update);
|
||||
this.handleResize();
|
||||
let camera_main = findCameraObjectUnderObject3D(
|
||||
this.state.scene_tree.find(["Cameras", "Main Camera"]).object
|
||||
);
|
||||
camera_main.updateProjectionMatrix();
|
||||
if (this.state.controls_main != null) {
|
||||
this.state.controls_main.update();
|
||||
}
|
||||
this.state.renderer_main.render(this.state.scene, camera_main);
|
||||
if (this.state.websocket.readyState === WebSocket.OPEN) {
|
||||
// update the camera information in the python server
|
||||
let cmd = "set_object";
|
||||
let path = "Cameras/Main Camera";
|
||||
let data = {
|
||||
type: cmd,
|
||||
path: path,
|
||||
object: camera_main.toJSON(),
|
||||
};
|
||||
let message = msgpack.encode(data);
|
||||
this.state.websocket.send(message);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -155,16 +109,25 @@ export class Viewer extends Component {
|
||||
|
||||
set_object(path, object) {
|
||||
this.state.scene_tree.find(path.concat(["<object>"])).set_object(object);
|
||||
// add controls if object is a camera
|
||||
if (object instanceof THREE.Camera) {
|
||||
this.state.controls_main = new TrackballControls(
|
||||
object,
|
||||
this.state.renderer_main.domElement
|
||||
);
|
||||
this.state.controls_main.rotateSpeed = 2.0;
|
||||
this.state.controls_main.zoomSpeed = 0.3;
|
||||
this.state.controls_main.panSpeed = 0.2;
|
||||
this.state.controls_main.staticMoving = false; // false is default
|
||||
this.state.controls_main.target.set(0, 0, 0); // focus point of the controls
|
||||
this.state.controls_main.autoRotate = false;
|
||||
this.state.controls_main.update();
|
||||
}
|
||||
}
|
||||
|
||||
set_object_from_json(path, object_json) {
|
||||
let loader = new ExtensibleObjectLoader();
|
||||
loader.onTextureLoad = () => {
|
||||
this.set_dirty();
|
||||
};
|
||||
loader.parse(object_json, (obj) => {
|
||||
// console.log("ethan.");
|
||||
// console.log(obj.type);
|
||||
if (
|
||||
obj.geometry !== undefined &&
|
||||
obj.geometry.type === "BufferGeometry"
|
||||
@@ -175,15 +138,10 @@ export class Viewer extends Component {
|
||||
) {
|
||||
obj.geometry.computeVertexNormals();
|
||||
}
|
||||
} else if (obj.type.includes("Camera")) {
|
||||
console.log("uses camera !!");
|
||||
// this.set_camera(obj);
|
||||
// this.set_3d_pane_size();
|
||||
}
|
||||
obj.castShadow = true;
|
||||
obj.receiveShadow = true;
|
||||
this.set_object(path, obj);
|
||||
this.set_dirty();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -209,17 +167,11 @@ export class Viewer extends Component {
|
||||
}
|
||||
|
||||
handle_command(cmd) {
|
||||
console.log(cmd);
|
||||
|
||||
// convert binary serialization format back to JSON
|
||||
cmd = msgpack.decode(new Uint8Array(cmd));
|
||||
console.log(cmd);
|
||||
|
||||
// TODO(ethan): ignore these or remove status. maybe incorporate into a clean view
|
||||
if (cmd.type === "status") {
|
||||
return;
|
||||
}
|
||||
|
||||
// three js scene commands
|
||||
if (cmd.type === "set_object") {
|
||||
let path = split_path(cmd.path);
|
||||
this.set_object_from_json(path, cmd.object);
|
||||
@@ -232,59 +184,14 @@ export class Viewer extends Component {
|
||||
} else if (cmd.type === "set_property") {
|
||||
let path = split_path(cmd.path);
|
||||
this.set_property(path, cmd.property, cmd.value);
|
||||
} else if (cmd.type === "set_animation") {
|
||||
// TODO(ethan): implement animations
|
||||
console.error("Animations not implemented yet.");
|
||||
} else if (cmd.type === "set_image") {
|
||||
console.log("ethan: setting camera");
|
||||
// var url = URL.createObjectURL(
|
||||
// new Blob([cmd["image"].buffer], { type: "image/png" } /* (1) */)
|
||||
// );
|
||||
// console.log(url);
|
||||
// document.getElementById("my-img").src = url;
|
||||
var height = 300;
|
||||
var width = 200;
|
||||
// console.log(cmd["image"]);
|
||||
var texture = new THREE.DataTexture(
|
||||
cmd["image"],
|
||||
width,
|
||||
height,
|
||||
THREE.RGBFormat
|
||||
);
|
||||
// var texture = new THREE.TextureLoader().load("texture2.jpeg");
|
||||
// var texture = new THREE.TextureLoader().load(url);
|
||||
// console.log(texture);
|
||||
// console.log(this.backgroundMesh);
|
||||
this.backgroundMesh.material.map = texture;
|
||||
this.backgroundMesh.material.map.needsUpdate = true;
|
||||
this.backgroundMesh.material.needsUpdate = true;
|
||||
}
|
||||
this.set_dirty();
|
||||
|
||||
if (cmd.type === "set_transform" && cmd.path === "/Cameras/Main Camera R") {
|
||||
let camera_r = findCameraObjectUnderObject3D(
|
||||
this.state.scene_tree.find(["Cameras", "Main Camera R"]).object
|
||||
);
|
||||
camera_r.updateProjectionMatrix();
|
||||
this.state.renderer_save.render(this.state.scene, camera_r);
|
||||
let imgData = this.state.renderer_save.domElement.toDataURL();
|
||||
this.state.images.push(imgData.toString());
|
||||
// this.save_image(imgData);
|
||||
// web rtc commands
|
||||
else if (cmd.type === "answer") {
|
||||
let answer = cmd.data;
|
||||
this.state.pc.setRemoteDescription(answer);
|
||||
}
|
||||
}
|
||||
|
||||
save_scene() {}
|
||||
|
||||
load_scene() {}
|
||||
|
||||
// save_image() {}
|
||||
|
||||
add_camera_helper(path, camera) {
|
||||
let camera_copy = camera.clone();
|
||||
camera_copy.far = 1; // so it doesn't show so far away
|
||||
this.set_object(path, new THREE.CameraHelper(camera_copy));
|
||||
}
|
||||
|
||||
getViewportWidth() {
|
||||
return window.innerWidth - (window.innerWidth % 2);
|
||||
}
|
||||
@@ -293,14 +200,94 @@ export class Viewer extends Component {
|
||||
return window.innerHeight;
|
||||
}
|
||||
|
||||
setupWebRTC() {
|
||||
console.log("setting up WebRTC");
|
||||
// The iceServers config comes from the following URL:
|
||||
// https://www.metered.ca/tools/openrelay/
|
||||
this.state.pc = new RTCPeerConnection({
|
||||
iceServers: [
|
||||
{
|
||||
urls: "stun:openrelay.metered.ca:80",
|
||||
},
|
||||
{
|
||||
urls: "turn:openrelay.metered.ca:80",
|
||||
username: "openrelayproject",
|
||||
credential: "openrelayproject",
|
||||
},
|
||||
{
|
||||
urls: "turn:openrelay.metered.ca:443",
|
||||
username: "openrelayproject",
|
||||
credential: "openrelayproject",
|
||||
},
|
||||
{
|
||||
urls: "turn:openrelay.metered.ca:443?transport=tcp",
|
||||
username: "openrelayproject",
|
||||
credential: "openrelayproject",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
// connect video
|
||||
this.state.pc.addEventListener("track", function (evt) {
|
||||
if (evt.track.kind == "video") {
|
||||
console.log("setting event stream");
|
||||
document.getElementById("WebRTCVideo-video").srcObject = evt.streams[0];
|
||||
}
|
||||
});
|
||||
this.state.pc.addTransceiver("video", { direction: "recvonly" });
|
||||
|
||||
this.state.pc
|
||||
.createOffer()
|
||||
.then((offer) => {
|
||||
return this.state.pc.setLocalDescription(offer);
|
||||
})
|
||||
.then(() => {
|
||||
// wait for ICE gathering to complete
|
||||
return new Promise((resolve) => {
|
||||
if (this.state.pc.iceGatheringState === "complete") {
|
||||
resolve();
|
||||
} else {
|
||||
var checkState = () => {
|
||||
if (this.state.pc.iceGatheringState === "complete") {
|
||||
this.state.pc.removeEventListener(
|
||||
"icegatheringstatechange",
|
||||
checkState
|
||||
);
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
this.state.pc.addEventListener(
|
||||
"icegatheringstatechange",
|
||||
checkState
|
||||
);
|
||||
}
|
||||
});
|
||||
})
|
||||
.then(() => {
|
||||
var offer = this.state.pc.localDescription;
|
||||
console.log("sending the offer");
|
||||
let cmd = "offer";
|
||||
let path = "";
|
||||
let data = {
|
||||
type: cmd,
|
||||
path: path,
|
||||
data: {
|
||||
sdp: offer.sdp,
|
||||
type: offer.type,
|
||||
},
|
||||
};
|
||||
let message = msgpack.encode(data);
|
||||
this.state.websocket.send(message);
|
||||
});
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
// Open the websocket
|
||||
let endpoint = websocket_endpoint_from_url(window.location.href);
|
||||
this.state.websocket_is_opened = false;
|
||||
this.state.websocket = new WebSocket("ws://" + endpoint + "/");
|
||||
this.state.websocket.onopen = () => {
|
||||
console.log("connected");
|
||||
this.state.websocket_is_opened = true;
|
||||
console.log("websocket connected");
|
||||
this.setupWebRTC();
|
||||
};
|
||||
this.state.websocket.binaryType = "arraybuffer";
|
||||
// 'command' for updates to Viewer
|
||||
@@ -310,195 +297,50 @@ export class Viewer extends Component {
|
||||
};
|
||||
|
||||
// Get size of screen
|
||||
this.state.viewport_width = this.getViewportWidth(); // notice that this is half width
|
||||
this.state.viewport_width = this.getViewportWidth();
|
||||
this.state.viewport_height = this.getViewportHeight();
|
||||
|
||||
// Scene
|
||||
this.state.scene = new THREE.Scene();
|
||||
this.state.scene.background = new THREE.Color(0xffffff);
|
||||
|
||||
// GUI
|
||||
this.state.gui = new GUI();
|
||||
let scene_folder = this.state.gui.addFolder("Scene");
|
||||
scene_folder.open();
|
||||
this.state.scene_tree = new SceneNode(this.state.scene, scene_folder, () =>
|
||||
this.set_dirty()
|
||||
);
|
||||
// let save_folder = this.state.gui.addFolder("Save / Load / Capture");
|
||||
// save_folder.add(this, 'save_scene');
|
||||
// save_folder.add(this, 'load_scene');
|
||||
// save_folder.add(this, 'save_image');
|
||||
// this.state.gui.open();
|
||||
this.state.scene_tree = new SceneNode(this.state.scene, scene_folder);
|
||||
|
||||
// console.log(this.mount);
|
||||
|
||||
// Renderer Left
|
||||
this.state.renderer_l = new THREE.WebGLRenderer({ antialias: true });
|
||||
this.state.renderer_l.setPixelRatio(window.devicePixelRatio);
|
||||
this.state.renderer_l.setSize(
|
||||
this.state.viewport_width,
|
||||
(this.state.viewport_height * 2) / 3
|
||||
);
|
||||
this.state.renderer_l.domElement.style.border = "1px solid black";
|
||||
this.mount.appendChild(this.state.renderer_l.domElement);
|
||||
|
||||
// Renderer Right
|
||||
let div = document.createElement("div");
|
||||
this.state.renderer_r = new THREE.WebGLRenderer({ antialias: true });
|
||||
this.state.renderer_r.setPixelRatio(window.devicePixelRatio);
|
||||
this.state.renderer_r.setSize(
|
||||
this.state.viewport_width / 2,
|
||||
(this.state.viewport_height * 1) / 3
|
||||
);
|
||||
this.state.renderer_r.domElement.style.border = "1px solid black";
|
||||
this.state.renderer_r.domElement.style = "inline-block";
|
||||
div.appendChild(this.state.renderer_r.domElement);
|
||||
|
||||
this.state.renderer_orth = new THREE.WebGLRenderer({ antialias: true });
|
||||
this.state.renderer_orth.setPixelRatio(window.devicePixelRatio);
|
||||
this.state.renderer_orth.setSize(
|
||||
this.state.viewport_width / 2,
|
||||
(this.state.viewport_height * 1) / 3
|
||||
);
|
||||
this.state.renderer_orth.domElement.style.border = "1px solid black";
|
||||
this.state.renderer_orth.domElement.style = "inline-block";
|
||||
div.appendChild(this.state.renderer_orth.domElement);
|
||||
// div.display.style = "relative";
|
||||
|
||||
// NOTE(ethan): not sure if this is needed anymore?
|
||||
this.state.renderer_save = new THREE.WebGLRenderer({
|
||||
// Renderer main
|
||||
this.state.renderer_main = new THREE.WebGLRenderer({
|
||||
antialias: true,
|
||||
preserveDrawingBuffer: true,
|
||||
alpha: true,
|
||||
});
|
||||
this.state.renderer_save.setPixelRatio(1.0);
|
||||
this.state.renderer_save.setSize(960, 540);
|
||||
this.state.renderer_save.domElement.style.border = "1px solid black";
|
||||
this.state.renderer_save.domElement.style.display = "none";
|
||||
// div.appendChild(this.state.renderer_save.domElement);
|
||||
this.state.renderer_main.setPixelRatio(window.devicePixelRatio);
|
||||
this.state.renderer_main.setSize(
|
||||
this.state.viewport_width,
|
||||
this.state.viewport_height
|
||||
);
|
||||
this.state.renderer_main.domElement.style.border = "1px solid black";
|
||||
|
||||
this.mount.appendChild(div);
|
||||
let domElement = document.getElementById("canvas-container-main");
|
||||
domElement.append(this.state.renderer_main.domElement);
|
||||
|
||||
// console.log(this.mount);
|
||||
|
||||
this.mount.style.display = "block";
|
||||
|
||||
// Camera settings at https://stackoverflow.com/questions/46182845/field-of-view-aspect-ratio-view-matrix-from-projection-matrix-hmd-ost-calib/46195462
|
||||
// Camera left
|
||||
this.state.camera_l = new THREE.PerspectiveCamera(
|
||||
// Camera main
|
||||
this.state.camera_main = new THREE.PerspectiveCamera(
|
||||
120,
|
||||
this.state.viewport_width / this.state.viewport_height,
|
||||
0.01,
|
||||
100
|
||||
);
|
||||
this.state.camera_l.position.x = 5;
|
||||
this.state.camera_l.position.y = -5;
|
||||
this.state.camera_l.position.z = -5;
|
||||
this.state.camera_l.up = new THREE.Vector3(0, -1, 0);
|
||||
// Controls left
|
||||
this.state.controls_l = new TrackballControls(
|
||||
this.state.camera_l,
|
||||
this.state.renderer_l.domElement
|
||||
);
|
||||
this.state.controls_l.rotateSpeed = 2.0;
|
||||
this.state.controls_l.zoomSpeed = 0.3;
|
||||
this.state.controls_l.panSpeed = 0.2;
|
||||
this.state.controls_l.staticMoving = false; // false is default
|
||||
this.state.controls_l.target.set(0, 0, 0); // focus point of the controls
|
||||
this.state.controls_l.autoRotate = false;
|
||||
this.state.controls_l.update();
|
||||
|
||||
// Camera right
|
||||
this.state.camera_r = new THREE.PerspectiveCamera(
|
||||
120,
|
||||
this.state.viewport_width / (this.state.viewport_height / 2),
|
||||
0.01,
|
||||
100
|
||||
);
|
||||
this.state.camera_orth = new THREE.OrthographicCamera(
|
||||
-2,
|
||||
2,
|
||||
-2,
|
||||
2,
|
||||
0.01,
|
||||
100
|
||||
);
|
||||
// this.state.camera_r.position.x = 0;
|
||||
// this.state.camera_r.position.y = 0;
|
||||
// this.state.camera_r.position.z = 0;
|
||||
// this.state.camera_r.up = new THREE.Vector3(0, -1, 0);
|
||||
|
||||
var texture = new THREE.TextureLoader().load("texture.jpeg");
|
||||
this.backgroundMesh = new THREE.Mesh(
|
||||
new THREE.PlaneGeometry(2, 2),
|
||||
new THREE.MeshBasicMaterial({
|
||||
map: texture,
|
||||
})
|
||||
// material
|
||||
);
|
||||
// const geometry = new THREE.PlaneGeometry(1, 1);
|
||||
// const material = new THREE.MeshBasicMaterial({
|
||||
// color: 0xffff00,
|
||||
// side: THREE.DoubleSide,
|
||||
// });
|
||||
// const plane = new THREE.Mesh(geometry, material);
|
||||
this.backgroundMesh.material.depthTest = false;
|
||||
this.backgroundMesh.material.depthWrite = false;
|
||||
// Create your background scene
|
||||
this.backgroundScene = new THREE.Scene();
|
||||
this.backgroundCamera = new THREE.Camera();
|
||||
this.backgroundScene.add(this.backgroundCamera);
|
||||
this.backgroundScene.add(this.backgroundMesh);
|
||||
|
||||
this.set_object(["Cameras", "Main Camera L"], this.state.camera_l);
|
||||
this.set_object(["Cameras", "Main Camera R"], this.state.camera_r);
|
||||
this.set_object(["Cameras", "Main Camera Orth"], this.state.camera_orth);
|
||||
|
||||
// rotate the cameras to match the COLMAP format
|
||||
let rotY = new THREE.Matrix4();
|
||||
rotY.makeRotationY(Math.PI);
|
||||
let rotZ = new THREE.Matrix4();
|
||||
rotZ.makeRotationZ(Math.PI);
|
||||
let rot = rotY.multiply(rotZ);
|
||||
this.set_transform(["Cameras", "Main Camera R", "<object>"], rot.toArray());
|
||||
|
||||
// this.state.scene_tree.find(["Cameras", "Main Camera L"]).set_property("position", [5, -5, -5]);
|
||||
this.state.scene_tree
|
||||
.find(["Cameras", "Main Camera R"])
|
||||
.set_property("position", [-2, -2, -2]);
|
||||
|
||||
// Controls right
|
||||
// this.state.controls_r = new TrackballControls(this.state.scene_tree.find(["Cameras", "Main Camera R"]).object, this.state.renderer_r.domElement);
|
||||
// this.state.controls_r.rotateSpeed = 2.0;
|
||||
// this.state.controls_r.zoomSpeed = 0.3;
|
||||
// this.state.controls_r.panSpeed = 0.2;
|
||||
// this.state.controls_r.staticMoving = false; // false is default
|
||||
// this.state.controls_r.target.set(0, 0, 0); // focus point of the controls
|
||||
// this.state.controls_r.autoRotate = false;
|
||||
// this.state.controls_r.update();
|
||||
|
||||
this.set_transform(
|
||||
["Cameras", "Main Camera Orth", "<object>"],
|
||||
rot.toArray()
|
||||
);
|
||||
this.state.scene_tree
|
||||
.find(["Cameras", "Main Camera Orth"])
|
||||
.set_property("position", [-1, -1, -1]);
|
||||
|
||||
let camera_axes_l = new THREE.AxesHelper(1);
|
||||
let camera_axes_r = new THREE.AxesHelper(1);
|
||||
let camera_axes_orth = new THREE.AxesHelper(1);
|
||||
this.set_object(["Cameras", "Main Camera L", "Axes"], camera_axes_l);
|
||||
this.set_object(["Cameras", "Main Camera R", "Axes"], camera_axes_r);
|
||||
this.set_object(["Cameras", "Main Camera Orth", "Axes"], camera_axes_orth);
|
||||
this.state.camera_main.position.x = 5;
|
||||
this.state.camera_main.position.y = -5;
|
||||
this.state.camera_main.position.z = -5;
|
||||
this.state.camera_main.up = new THREE.Vector3(0, -1, 0);
|
||||
this.set_object(["Cameras", "Main Camera"], this.state.camera_main);
|
||||
|
||||
// Axes display
|
||||
let axes = new THREE.AxesHelper(5);
|
||||
this.set_object(["Axes"], axes);
|
||||
|
||||
// var arrowPos = new THREE.Vector3(0, 0, 0);
|
||||
// var arrow = new THREE.ArrowHelper(new THREE.Vector3(1, 0, 0), arrowPos, 60, 0xFF0000, 20, 10)
|
||||
// this.set_object(["arrow"], arrow);
|
||||
|
||||
// Grid
|
||||
let grid = new THREE.GridHelper(20, 20);
|
||||
this.set_object(["Grid"], grid);
|
||||
@@ -511,32 +353,23 @@ export class Viewer extends Component {
|
||||
|
||||
// Stats
|
||||
var stats = new Stats();
|
||||
this.mount.appendChild(stats.dom);
|
||||
let domElement_stats = document.getElementById("canvas-container-main");
|
||||
domElement_stats.append(stats.dom);
|
||||
|
||||
// let camera = new THREE.PerspectiveCamera(75, this.state.width / this.state.height, 0.1, 1000);
|
||||
// let helper = new THREE.CameraHelper(camera);
|
||||
// // scene.add( helper );
|
||||
// // console.log(helper);
|
||||
// console.log(helper);
|
||||
// // this.set_object(["Helper"], helper);
|
||||
// // console.log(this.state.scene);
|
||||
|
||||
// Cube
|
||||
// var geometry = new THREE.BoxGeometry(1, 1, 1);
|
||||
// var material = new THREE.MeshBasicMaterial({color: 0x00ff00});
|
||||
// var cube = new THREE.Mesh(geometry, material);
|
||||
// this.state.scene.add(cube);
|
||||
|
||||
this.animate();
|
||||
}
|
||||
|
||||
componentWillUnmount() {
|
||||
// this.mount.removeChild(this.state.renderer_l.domElement);
|
||||
// this.mount.removeChild(this.state.renderer_r.domElement);
|
||||
removeAllChildNodes(this.mount);
|
||||
this.update();
|
||||
}
|
||||
|
||||
render() {
|
||||
return <div ref={(ref) => (this.mount = ref)} />;
|
||||
return (
|
||||
<div>
|
||||
<div className="WebRTCVideo">
|
||||
<video id="WebRTCVideo-video" autoPlay playsInline muted>
|
||||
{" "}
|
||||
</video>{" "}
|
||||
</div>{" "}
|
||||
<div id="canvas-container-main"> </div>{" "}
|
||||
<div id="stats-container"> </div>{" "}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,43 @@
|
||||
body {
|
||||
margin: 0;
|
||||
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
|
||||
'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen",
|
||||
"Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue",
|
||||
sans-serif;
|
||||
-webkit-font-smoothing: antialiased;
|
||||
-moz-osx-font-smoothing: grayscale;
|
||||
}
|
||||
|
||||
code {
|
||||
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
|
||||
font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New",
|
||||
monospace;
|
||||
}
|
||||
|
||||
.BackgroundDiv,
|
||||
.WebRTCVideo {
|
||||
position: absolute;
|
||||
top: 0px;
|
||||
right: 0px;
|
||||
bottom: 0px;
|
||||
left: 0px;
|
||||
}
|
||||
|
||||
#BackgroundDiv-video,
|
||||
#WebRTCVideo-video {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: fill;
|
||||
}
|
||||
|
||||
#canvas-container-main {
|
||||
position: absolute;
|
||||
top: 0px;
|
||||
right: 0px;
|
||||
bottom: 0px;
|
||||
left: 0px;
|
||||
}
|
||||
|
||||
#canvas-container-main canvas {
|
||||
position: relative;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
}
|
||||
@@ -1,17 +1,13 @@
|
||||
import React from 'react';
|
||||
import React from "react";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import './index.css';
|
||||
import App from './App';
|
||||
import * as serviceWorker from './serviceWorker';
|
||||
import "./index.css";
|
||||
import * as serviceWorker from "./serviceWorker";
|
||||
import { Viewer } from "./Viewer";
|
||||
|
||||
const root = ReactDOM.createRoot(document.getElementById("root"));
|
||||
root.render(
|
||||
<React.StrictMode>
|
||||
<App />
|
||||
</React.StrictMode>
|
||||
);
|
||||
root.render( < Viewer / > );
|
||||
|
||||
// If you want your app to work offline and load faster, you can change
|
||||
// unregister() to register() below. Note this comes with some pitfalls.
|
||||
// Learn more about service workers: https://bit.ly/CRA-PWA
|
||||
serviceWorker.unregister();
|
||||
serviceWorker.unregister();
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,48 @@
|
||||
function createPeerConnection() {
|
||||
var config = {
|
||||
sdpSemantics: "unified-plan",
|
||||
};
|
||||
|
||||
// if (document.getElementById('use-stun').checked) {
|
||||
// config.iceServers = [{urls: ['stun:stun.l.google.com:19302']}];
|
||||
// }
|
||||
|
||||
let pc = new RTCPeerConnection(config);
|
||||
|
||||
// register some listeners to help debugging
|
||||
pc.addEventListener(
|
||||
"icegatheringstatechange",
|
||||
function () {
|
||||
iceGatheringLog.textContent += " -> " + pc.iceGatheringState;
|
||||
},
|
||||
false
|
||||
);
|
||||
iceGatheringLog.textContent = pc.iceGatheringState;
|
||||
|
||||
pc.addEventListener(
|
||||
"iceconnectionstatechange",
|
||||
function () {
|
||||
iceConnectionLog.textContent += " -> " + pc.iceConnectionState;
|
||||
},
|
||||
false
|
||||
);
|
||||
iceConnectionLog.textContent = pc.iceConnectionState;
|
||||
|
||||
pc.addEventListener(
|
||||
"signalingstatechange",
|
||||
function () {
|
||||
signalingLog.textContent += " -> " + pc.signalingState;
|
||||
},
|
||||
false
|
||||
);
|
||||
signalingLog.textContent = pc.signalingState;
|
||||
|
||||
// connect audio / video
|
||||
pc.addEventListener("track", function (evt) {
|
||||
if (evt.track.kind == "video")
|
||||
document.getElementById("video").srcObject = evt.streams[0];
|
||||
else document.getElementById("audio").srcObject = evt.streams[0];
|
||||
});
|
||||
|
||||
return pc;
|
||||
}
|
||||
@@ -18,7 +18,6 @@ from . import commands
|
||||
from . import geometry
|
||||
from . import visualizer
|
||||
from . import transformations
|
||||
from . import animation
|
||||
from .visualizer import ViewerWindow, Visualizer
|
||||
|
||||
|
||||
|
||||
@@ -1,184 +0,0 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import tempfile
|
||||
import tarfile
|
||||
import sys
|
||||
import os.path
|
||||
import subprocess
|
||||
|
||||
if sys.version_info >= (3, 0):
|
||||
unicode = str
|
||||
|
||||
import bisect
|
||||
from . import transformations as tf
|
||||
|
||||
|
||||
class AnimationTrack(object):
|
||||
__slots__ = ["name", "jstype", "frames", "values"]
|
||||
|
||||
def __init__(self, name, jstype, frames=None, values=None):
|
||||
self.name = name
|
||||
self.jstype = jstype
|
||||
if frames is None:
|
||||
self.frames = []
|
||||
else:
|
||||
self.frames = frames
|
||||
if values is None:
|
||||
self.values = []
|
||||
else:
|
||||
self.values = values
|
||||
|
||||
def set_property(self, frame, value):
|
||||
i = bisect.bisect(self.frames, frame)
|
||||
self.frames.insert(i, frame)
|
||||
self.values.insert(i, value)
|
||||
|
||||
def lower(self):
|
||||
return {
|
||||
"name": unicode("." + self.name),
|
||||
"type": unicode(self.jstype),
|
||||
"keys": [{"time": self.frames[i], "value": self.values[i]} for i in range(len(self.frames))],
|
||||
}
|
||||
|
||||
|
||||
class AnimationClip(object):
|
||||
__slots__ = ["tracks", "fps", "name"]
|
||||
|
||||
def __init__(self, tracks=None, fps=30, name="default"):
|
||||
if tracks is None:
|
||||
self.tracks = {}
|
||||
else:
|
||||
self.tracks = tracks
|
||||
self.fps = fps
|
||||
self.name = name
|
||||
|
||||
def set_property(self, frame, property, jstype, value):
|
||||
if property not in self.tracks:
|
||||
self.tracks[property] = AnimationTrack(property, jstype)
|
||||
track = self.tracks[property]
|
||||
track.set_property(frame, value)
|
||||
|
||||
def lower(self):
|
||||
return {"fps": self.fps, "name": unicode(self.name), "tracks": [t.lower() for t in self.tracks.values()]}
|
||||
|
||||
|
||||
class Animation(object):
|
||||
__slots__ = ["clips", "default_framerate"]
|
||||
|
||||
def __init__(self, clips=None, default_framerate=30):
|
||||
if clips is None:
|
||||
self.clips = {}
|
||||
else:
|
||||
self.clips = clips
|
||||
self.default_framerate = default_framerate
|
||||
|
||||
def lower(self):
|
||||
return [{"path": path.lower(), "clip": clip.lower()} for (path, clip) in self.clips.items()]
|
||||
|
||||
def at_frame(self, visualizer, frame):
|
||||
return AnimationFrameVisualizer(self, visualizer.path, frame)
|
||||
|
||||
|
||||
def js_position(matrix):
|
||||
return list(matrix[:3, 3])
|
||||
|
||||
|
||||
def js_quaternion(matrix):
|
||||
quat = tf.quaternion_from_matrix(matrix)
|
||||
return [quat[1], quat[2], quat[3], quat[0]]
|
||||
|
||||
|
||||
class AnimationFrameVisualizer(object):
|
||||
__slots__ = ["animation", "path", "current_frame"]
|
||||
|
||||
def __init__(self, animation, path, current_frame):
|
||||
self.animation = animation
|
||||
self.path = path
|
||||
self.current_frame = current_frame
|
||||
|
||||
def get_clip(self):
|
||||
if self.path not in self.animation.clips:
|
||||
self.animation.clips[self.path] = AnimationClip(fps=self.animation.default_framerate)
|
||||
return self.animation.clips[self.path]
|
||||
|
||||
def set_transform(self, matrix):
|
||||
assert matrix.shape == (4, 4)
|
||||
clip = self.get_clip()
|
||||
clip.set_property(self.current_frame, "position", "vector3", js_position(matrix))
|
||||
clip.set_property(self.current_frame, "quaternion", "quaternion", js_quaternion(matrix))
|
||||
|
||||
def set_property(self, prop, jstype, value):
|
||||
clip = self.get_clip()
|
||||
clip.set_property(self.current_frame, prop, jstype, value)
|
||||
|
||||
def __getitem__(self, path):
|
||||
return AnimationFrameVisualizer(self.animation, self.path.append(path), self.current_frame)
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *arg):
|
||||
pass
|
||||
|
||||
|
||||
def convert_frames_to_video(tar_file_path, output_path="output.mp4", framerate=60, overwrite=False):
|
||||
"""
|
||||
Try to convert a tar file containing a sequence of frames saved by the
|
||||
pyrad viewer into a single video file.
|
||||
|
||||
This relies on having `ffmpeg` installed on your system.
|
||||
"""
|
||||
output_path = os.path.abspath(output_path)
|
||||
if os.path.isfile(output_path) and not overwrite:
|
||||
raise ValueError(
|
||||
"The output path {:s} already exists. To overwrite that file, you can pass overwrite=True to this function.".format(
|
||||
output_path
|
||||
)
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
with tarfile.open(tar_file_path) as tar:
|
||||
tar.extractall(tmp_dir)
|
||||
args = [
|
||||
"ffmpeg",
|
||||
"-r",
|
||||
str(framerate),
|
||||
"-i",
|
||||
r"%07d.png",
|
||||
"-vcodec",
|
||||
"libx264",
|
||||
"-preset",
|
||||
"slow",
|
||||
"-crf",
|
||||
"18",
|
||||
]
|
||||
if overwrite:
|
||||
args.append("-y")
|
||||
args.append(output_path)
|
||||
try:
|
||||
subprocess.check_call(args, cwd=tmp_dir)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(
|
||||
"""
|
||||
Could not call `ffmpeg` to convert your frames into a video.
|
||||
If you want to convert the frames manually, you can extract the
|
||||
.tar archive into a directory, cd to that directory, and run:
|
||||
ffmpeg -r 60 -i %07d.png \\\n\t -vcodec libx264 \\\n\t -preset slow \\\n\t -crf 18 \\\n\t output.mp4
|
||||
"""
|
||||
)
|
||||
raise
|
||||
print("Saved output as {:s}".format(output_path))
|
||||
return output_path
|
||||
@@ -1,49 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
|
||||
def file_filter(name):
|
||||
return (not name.startswith(".")) and (not name.endswith(".swp"))
|
||||
|
||||
|
||||
def file_times(path):
|
||||
for top_level in filter(file_filter, os.listdir(path)):
|
||||
for root, dirs, files in os.walk(top_level):
|
||||
for file in filter(file_filter, files):
|
||||
yield os.stat(os.path.join(root, file)).st_mtime
|
||||
|
||||
|
||||
def print_stdout(process):
|
||||
stdout = process.stdout
|
||||
if stdout != None:
|
||||
print(stdout)
|
||||
|
||||
|
||||
# We concatenate all of the arguments together, and treat that as the command to run
|
||||
command = " ".join(sys.argv[1:])
|
||||
|
||||
# The path to watch
|
||||
path = "."
|
||||
|
||||
# How often we check the filesystem for changes (in seconds)
|
||||
wait = 1
|
||||
|
||||
# The process to autoreload
|
||||
process = subprocess.Popen(command, shell=True)
|
||||
|
||||
# The current maximum file modified time under the watched directory
|
||||
last_mtime = max(file_times(path))
|
||||
|
||||
|
||||
while True:
|
||||
max_mtime = max(file_times(path))
|
||||
print_stdout(process)
|
||||
if max_mtime > last_mtime:
|
||||
last_mtime = max_mtime
|
||||
process.kill()
|
||||
print("Restarting process.")
|
||||
process = subprocess.Popen(command, shell=True)
|
||||
time.sleep(wait)
|
||||
@@ -19,83 +19,6 @@ from . import geometry as g
|
||||
import numpy as np
|
||||
import cv2
|
||||
|
||||
"""
|
||||
TODO(ethan): add some notes about the coordinate conventions of the cameras
|
||||
"""
|
||||
|
||||
|
||||
class Camera(SceneElement):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
|
||||
class OrthographicCamera(Camera):
|
||||
def __init__(self, left, right, top, bottom, near, far, zoom=1):
|
||||
super().__init__()
|
||||
self.left = left
|
||||
self.right = right
|
||||
self.top = top
|
||||
self.bottom = bottom
|
||||
self.near = near
|
||||
self.far = far
|
||||
self.zoom = zoom
|
||||
|
||||
def lower(self):
|
||||
data = {
|
||||
"object": {
|
||||
"uuid": self.uuid,
|
||||
"type": "OrthographicCamera",
|
||||
"left": self.left,
|
||||
"right": self.right,
|
||||
"top": self.top,
|
||||
"bottom": self.bottom,
|
||||
"near": self.near,
|
||||
"far": self.far,
|
||||
"zoom": self.zoom,
|
||||
}
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
class PerspectiveCamera(Camera):
|
||||
def __init__(self, aspect, fov, near, far, zoom=1):
|
||||
super().__init__()
|
||||
self.aspect = aspect
|
||||
self.fov = fov
|
||||
self.near = near
|
||||
self.far = far
|
||||
|
||||
@staticmethod
|
||||
def from_intrinsics(focal_length, aspect_ratio):
|
||||
"""TODO(ethan): finish this"""
|
||||
# convert the intrinsics matrix into three.js compatible format
|
||||
return PerspectiveCamera(aspect, fov, near, far)
|
||||
|
||||
def lower(self):
|
||||
data = {
|
||||
"object": {
|
||||
"uuid": self.uuid,
|
||||
"type": "PerspectiveCamera",
|
||||
"aspect": self.aspect,
|
||||
"fov": self.fov,
|
||||
"near": self.near,
|
||||
"far": self.far,
|
||||
}
|
||||
}
|
||||
return data
|
||||
|
||||
|
||||
class CameraHelper(Camera):
|
||||
"""NOTE(ethan): Camera parent class might not make sense here?"""
|
||||
|
||||
def __init__(self, camera):
|
||||
super().__init__()
|
||||
self.camera = camera
|
||||
|
||||
def lower(self):
|
||||
data = {"object": {"uuid": self.uuid, "type": "CameraHelper", "camera": self.camera.lower()}}
|
||||
return data
|
||||
|
||||
|
||||
class ImagePlane(g.Mesh):
|
||||
def __init__(self, image, height=1, width=1):
|
||||
|
||||
@@ -29,7 +29,6 @@ from .geometry import (
|
||||
PointsMaterial,
|
||||
Points,
|
||||
)
|
||||
from .cameras import Camera, CameraHelper
|
||||
from .path import Path
|
||||
|
||||
|
||||
@@ -59,6 +58,36 @@ class SetObject:
|
||||
return {"type": "set_object", "object": self.object.lower(), "path": self.path.lower()}
|
||||
|
||||
|
||||
class GetObject:
|
||||
__slots__ = ["path"]
|
||||
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
def lower(self):
|
||||
return {"type": "get_object", "path": self.path.lower()}
|
||||
|
||||
|
||||
class SetImage:
|
||||
"""
|
||||
Sets the image for showing with web rtc.
|
||||
"""
|
||||
|
||||
__slots__ = ["image", "path"]
|
||||
|
||||
def __init__(self, image, path):
|
||||
self.image = image
|
||||
self.path = path
|
||||
|
||||
def lower(self):
|
||||
return {
|
||||
"type": "set_image",
|
||||
"path": self.path.lower(),
|
||||
"image": list(self.image.flatten()),
|
||||
"shape": list(self.image.shape),
|
||||
}
|
||||
|
||||
|
||||
class SetTransform:
|
||||
__slots__ = ["matrix", "path"]
|
||||
|
||||
@@ -90,31 +119,3 @@ class SetProperty:
|
||||
|
||||
def lower(self):
|
||||
return {"type": "set_property", "path": self.path.lower(), "property": self.key.lower(), "value": self.value}
|
||||
|
||||
|
||||
class SetAnimation:
|
||||
__slots__ = ["animation", "play", "repetitions"]
|
||||
|
||||
def __init__(self, animation, play=True, repetitions=1):
|
||||
self.animation = animation
|
||||
self.play = play
|
||||
self.repetitions = repetitions
|
||||
|
||||
def lower(self):
|
||||
return {
|
||||
"type": "set_animation",
|
||||
"animations": self.animation.lower(),
|
||||
"options": {"play": self.play, "repetitions": self.repetitions},
|
||||
"path": "",
|
||||
}
|
||||
|
||||
|
||||
class SetImage:
|
||||
__slots__ = ["image", "path"]
|
||||
|
||||
def __init__(self, image, path):
|
||||
self.image = image
|
||||
self.path = path
|
||||
|
||||
def lower(self):
|
||||
return {"type": "set_image", "image": self.image, "path": self.path.lower()}
|
||||
@@ -15,8 +15,10 @@
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import numpy as np
|
||||
|
||||
if sys.version_info >= (3, 0):
|
||||
ADDRESS_IN_USE_ERROR = OSError
|
||||
@@ -29,11 +31,16 @@ import tornado.gen
|
||||
import tornado.ioloop
|
||||
import tornado.web
|
||||
import tornado.websocket
|
||||
import umsgpack
|
||||
import zmq
|
||||
import zmq.eventloop.ioloop
|
||||
from pyrad.viewer.backend.tree import SceneTree, find_node, walk
|
||||
from aiortc import RTCPeerConnection, RTCSessionDescription, VideoStreamTrack
|
||||
from aiortc.rtcrtpsender import RTCRtpSender
|
||||
from zmq.eventloop.zmqstream import ZMQStream
|
||||
|
||||
from pyrad.viewer.backend.tree import SceneTree, find_node, walk
|
||||
from pyrad.viewer.backend.video_stream import FlagVideoStreamTrack, SingleFrameStreamTrack
|
||||
|
||||
|
||||
def capture(pattern, s):
|
||||
match = re.match(pattern, s)
|
||||
@@ -66,7 +73,8 @@ MAX_ATTEMPTS = 1000
|
||||
DEFAULT_ZMQ_METHOD = "tcp"
|
||||
DEFAULT_ZMQ_PORT = 6000
|
||||
DEFAULT_WEBSOCKET_PORT = 8051
|
||||
MESHCAT_COMMANDS = ["set_transform", "set_object", "delete", "set_property", "set_animation", "set_image"]
|
||||
WEBSOCKET_COMMANDS = ["set_transform", "set_object", "get_object", "set_property", "delete"]
|
||||
WEBRTC_COMMANDS = ["set_image"]
|
||||
|
||||
|
||||
def find_available_port(func, default_port, max_attempts=MAX_ATTEMPTS, **kwargs):
|
||||
@@ -89,6 +97,13 @@ def find_available_port(func, default_port, max_attempts=MAX_ATTEMPTS, **kwargs)
|
||||
)
|
||||
|
||||
|
||||
def force_codec(pc, sender, forced_codec):
|
||||
kind = forced_codec.split("/")[0]
|
||||
codecs = RTCRtpSender.getCapabilities(kind).codecs
|
||||
transceiver = next(t for t in pc.getTransceivers() if t.sender == sender)
|
||||
transceiver.setCodecPreferences([codec for codec in codecs if codec.mimeType == forced_codec])
|
||||
|
||||
|
||||
class WebSocketHandler(tornado.websocket.WebSocketHandler):
|
||||
def __init__(self, *args, **kwargs):
|
||||
self.bridge = kwargs.pop("bridge")
|
||||
@@ -103,25 +118,51 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
|
||||
print("opened:", self, file=sys.stderr)
|
||||
self.bridge.send_scene(self)
|
||||
|
||||
def on_message(self, message):
|
||||
print(message)
|
||||
async def on_message(self, message):
|
||||
|
||||
data = message
|
||||
m = umsgpack.unpackb(message)
|
||||
type_ = m["type"]
|
||||
path = list(filter(lambda x: len(x) > 0, m["path"].split("/")))
|
||||
|
||||
if type_ == "set_transform":
|
||||
find_node(self.bridge.tree, path).transform = data
|
||||
elif type_ == "set_object":
|
||||
find_node(self.bridge.tree, path).object = data
|
||||
find_node(self.bridge.tree, path).properties = []
|
||||
elif type_ == "offer":
|
||||
print("making an offer")
|
||||
print("sending an answer")
|
||||
# print(m)
|
||||
|
||||
offer = RTCSessionDescription(m["data"]["sdp"], m["data"]["type"])
|
||||
|
||||
pc = RTCPeerConnection()
|
||||
self.bridge.pcs.add(pc) # TODO(ethan): handle this better, since this set will get large
|
||||
|
||||
# video = FlagVideoStreamTrack()
|
||||
video = SingleFrameStreamTrack()
|
||||
self.bridge.video_tracks.add(video)
|
||||
video_sender = pc.addTrack(video)
|
||||
# force_codec(this.bridge.pc, video_sender, video_codec)
|
||||
|
||||
await pc.setRemoteDescription(offer)
|
||||
answer = await pc.createAnswer()
|
||||
await pc.setLocalDescription(answer)
|
||||
|
||||
cmd_data = {
|
||||
"type": "answer",
|
||||
"path": "",
|
||||
"data": {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type},
|
||||
}
|
||||
data = umsgpack.packb(cmd_data)
|
||||
self.write_message(data, binary=True)
|
||||
|
||||
def on_close(self):
|
||||
self.bridge.websocket_pool.remove(self)
|
||||
print("closed:", self, file=sys.stderr)
|
||||
|
||||
|
||||
def create_command(data):
|
||||
"""Encode the drawing command into a Javascript fetch() command for display."""
|
||||
return """
|
||||
fetch("data:application/octet-binary;base64,{}")
|
||||
.then(res => res.arrayBuffer())
|
||||
.then(buffer => viewer.handle_command_bytearray(new Uint8Array(buffer)));
|
||||
""".format(
|
||||
base64.b64encode(data).decode("utf-8")
|
||||
)
|
||||
|
||||
|
||||
class ZMQWebSocketBridge(object):
|
||||
context = zmq.Context()
|
||||
|
||||
@@ -130,6 +171,8 @@ class ZMQWebSocketBridge(object):
|
||||
self.websocket_pool = set()
|
||||
self.app = self.make_app()
|
||||
self.ioloop = tornado.ioloop.IOLoop.current()
|
||||
self.pcs = set()
|
||||
self.video_tracks = set()
|
||||
|
||||
if zmq_url is None:
|
||||
|
||||
@@ -155,44 +198,58 @@ class ZMQWebSocketBridge(object):
|
||||
return f"{class_name} using zmq_url={self.zmq_url} and websocket_port={self.websocket_port}"
|
||||
|
||||
def make_app(self):
|
||||
"""Create a tornado application for the websocket server."""
|
||||
return tornado.web.Application([(r"/", WebSocketHandler, {"bridge": self})])
|
||||
|
||||
def handle_zmq(self, frames):
|
||||
cmd = frames[0].decode("utf-8")
|
||||
if cmd not in MESHCAT_COMMANDS:
|
||||
print(f"{cmd} not in MESHCAT_COMMANDS! Proceeding anyways.")
|
||||
# self.zmq_socket.send(b"error: unrecognized comand")
|
||||
print(cmd)
|
||||
if len(frames) != 3:
|
||||
self.zmq_socket.send(b"error: expected 3 frames")
|
||||
return
|
||||
path = list(filter(lambda x: len(x) > 0, frames[1].decode("utf-8").split("/")))
|
||||
data = frames[2]
|
||||
self.forward_to_websockets(frames)
|
||||
if cmd == "set_transform":
|
||||
find_node(self.tree, path).transform = data
|
||||
elif cmd == "set_object":
|
||||
find_node(self.tree, path).object = data
|
||||
find_node(self.tree, path).properties = []
|
||||
elif cmd == "set_property":
|
||||
find_node(self.tree, path).properties.append(data)
|
||||
elif cmd == "set_animation":
|
||||
find_node(self.tree, path).animation = data
|
||||
elif cmd == "delete":
|
||||
if len(path) > 0:
|
||||
parent = find_node(self.tree, path[:-1])
|
||||
child = path[-1]
|
||||
if child in parent:
|
||||
del parent[child]
|
||||
else:
|
||||
self.tree = SceneTree()
|
||||
if cmd in WEBSOCKET_COMMANDS:
|
||||
self.forward_to_websockets(frames)
|
||||
if cmd == "set_transform":
|
||||
find_node(self.tree, path).transform = data
|
||||
elif cmd == "set_object":
|
||||
find_node(self.tree, path).object = data
|
||||
find_node(self.tree, path).properties = []
|
||||
elif cmd == "get_object":
|
||||
data = find_node(self.tree, path).object
|
||||
self.zmq_socket.send(data)
|
||||
return
|
||||
elif cmd == "set_property":
|
||||
find_node(self.tree, path).properties.append(data)
|
||||
elif cmd == "delete":
|
||||
if len(path) > 0:
|
||||
parent = find_node(self.tree, path[:-1])
|
||||
child = path[-1]
|
||||
if child in parent:
|
||||
del parent[child]
|
||||
else:
|
||||
self.tree = SceneTree()
|
||||
elif cmd in WEBRTC_COMMANDS:
|
||||
if cmd == "set_image":
|
||||
for video_track in self.video_tracks:
|
||||
unpacked_data = umsgpack.unpackb(data)
|
||||
image = np.array(unpacked_data["image"], dtype="uint8").reshape(unpacked_data["shape"])
|
||||
video_track.put_frame(image)
|
||||
else:
|
||||
self.zmq_socket.send(b"error: unknown command")
|
||||
return
|
||||
self.zmq_socket.send(b"ok")
|
||||
return
|
||||
|
||||
def forward_to_websockets(self, frames):
|
||||
cmd, path, data = frames
|
||||
"""Forward a zmq message to all websockets."""
|
||||
_, _, data = frames # cmd, path, data
|
||||
for websocket in self.websocket_pool:
|
||||
websocket.write_message(data, binary=True)
|
||||
|
||||
def setup_zmq(self, url):
|
||||
"""Setup a zmq socket and connect it to the given url."""
|
||||
zmq_socket = self.context.socket(zmq.REP)
|
||||
zmq_socket.bind(url)
|
||||
zmq_stream = ZMQStream(zmq_socket)
|
||||
@@ -207,8 +264,6 @@ class ZMQWebSocketBridge(object):
|
||||
websocket.write_message(p, binary=True)
|
||||
if node.transform is not None:
|
||||
websocket.write_message(node.transform, binary=True)
|
||||
if node.animation is not None:
|
||||
websocket.write_message(node.animation, binary=True)
|
||||
|
||||
def run(self):
|
||||
self.ioloop.start()
|
||||
@@ -217,7 +272,7 @@ class ZMQWebSocketBridge(object):
|
||||
def main():
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description="Serve the MeshCat HTML files and listen for ZeroMQ commands")
|
||||
parser = argparse.ArgumentParser(description="Listen for ZeroMQ commands")
|
||||
parser.add_argument("--zmq-url", "-z", type=str, nargs="?", default=None)
|
||||
parser.add_argument("--websocket-port", "-wp", type=str, nargs="?", default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
|
||||
def get_chunks(lst, num_chunks=None, size_of_chunk=None):
|
||||
"""Returns list of n elements, constaining a sublist."""
|
||||
if num_chunks:
|
||||
assert not size_of_chunk
|
||||
size = len(lst) // num_chunks
|
||||
if size_of_chunk:
|
||||
assert not num_chunks
|
||||
size = size_of_chunk
|
||||
chunks = []
|
||||
for i in range(0, len(lst), size):
|
||||
chunks.append(lst[i : i + size])
|
||||
return chunks
|
||||
|
||||
|
||||
def get_intrinsics_matrix_and_camera_to_world_h(camera_object, image_height):
|
||||
"""Returns the camera intrinsics matrix and the camera to world homogeneous matrix.
|
||||
Args:
|
||||
camera_object: a Camera object.
|
||||
image_size: the size of the image (height, width)
|
||||
"""
|
||||
# intrinsics
|
||||
fov = camera_object["fov"]
|
||||
aspect = camera_object["aspect"]
|
||||
image_width = aspect * image_height
|
||||
pp_w = image_width / 2.0
|
||||
pp_h = image_height / 2.0
|
||||
focal_length = pp_h / np.tan(fov * (np.pi / 180.0) / 2.0)
|
||||
intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float()
|
||||
|
||||
# extrinsics
|
||||
camera_to_world_h = torch.tensor(get_chunks(camera_object["matrix"], size_of_chunk=4)).T.float()
|
||||
|
||||
return intrinsics_matrix, camera_to_world_h
|
||||
@@ -0,0 +1,93 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import math
|
||||
|
||||
import cv2
|
||||
import numpy
|
||||
from aiortc import VideoStreamTrack
|
||||
from av import VideoFrame
|
||||
|
||||
|
||||
class FlagVideoStreamTrack(VideoStreamTrack):
|
||||
"""
|
||||
A video track that returns an animated flag.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__() # don't forget this!
|
||||
self.counter = 0
|
||||
height, width = 480, 640
|
||||
|
||||
# generate flag
|
||||
data_bgr = numpy.hstack(
|
||||
[
|
||||
self._create_rectangle(width=213, height=480, color=(255, 0, 0)), # blue
|
||||
self._create_rectangle(width=214, height=480, color=(255, 255, 255)), # white
|
||||
self._create_rectangle(width=213, height=480, color=(0, 0, 255)), # red
|
||||
]
|
||||
)
|
||||
|
||||
# shrink and center it
|
||||
M = numpy.float32([[0.5, 0, width / 4], [0, 0.5, height / 4]])
|
||||
data_bgr = cv2.warpAffine(data_bgr, M, (width, height))
|
||||
|
||||
# compute animation
|
||||
omega = 2 * math.pi / height
|
||||
id_x = numpy.tile(numpy.array(range(width), dtype=numpy.float32), (height, 1))
|
||||
id_y = numpy.tile(numpy.array(range(height), dtype=numpy.float32), (width, 1)).transpose()
|
||||
|
||||
self.frames = []
|
||||
for k in range(30):
|
||||
phase = 2 * k * math.pi / 30
|
||||
map_x = id_x + 10 * numpy.cos(omega * id_x + phase)
|
||||
map_y = id_y + 10 * numpy.sin(omega * id_x + phase)
|
||||
frame = cv2.remap(data_bgr, map_x, map_y, cv2.INTER_LINEAR)
|
||||
self.frames.append(VideoFrame.from_ndarray(frame, format="bgr24"))
|
||||
|
||||
async def recv(self):
|
||||
pts, time_base = await self.next_timestamp()
|
||||
|
||||
frame = self.frames[self.counter % 30]
|
||||
frame.pts = pts
|
||||
frame.time_base = time_base
|
||||
self.counter += 1
|
||||
return frame
|
||||
|
||||
def _create_rectangle(self, width, height, color):
|
||||
data_bgr = numpy.zeros((height, width, 3), numpy.uint8)
|
||||
data_bgr[:, :] = color
|
||||
return data_bgr
|
||||
|
||||
|
||||
class SingleFrameStreamTrack(VideoStreamTrack):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.background_frame = numpy.zeros((480, 640, 3), dtype="uint8") * 255
|
||||
self.frame = None
|
||||
self.put_frame(self.background_frame)
|
||||
|
||||
def put_frame(self, frame):
|
||||
self.frame = VideoFrame.from_ndarray(frame)
|
||||
|
||||
async def recv(self):
|
||||
|
||||
pts, time_base = await self.next_timestamp()
|
||||
|
||||
frame = self.frame
|
||||
frame.pts = pts
|
||||
frame.time_base = time_base
|
||||
return frame
|
||||
@@ -0,0 +1,213 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Client class"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import ssl
|
||||
|
||||
import aiohttp_cors
|
||||
import cv2
|
||||
import numpy
|
||||
from aiohttp import web
|
||||
from aiortc import RTCPeerConnection, RTCSessionDescription, VideoStreamTrack
|
||||
from aiortc.rtcrtpsender import RTCRtpSender
|
||||
from av import VideoFrame
|
||||
|
||||
ROOT = os.path.dirname(__file__)
|
||||
|
||||
|
||||
class SingleFrameStreamTrack(VideoStreamTrack):
|
||||
def __init__(self):
|
||||
super().__init__() # don't forget this!
|
||||
|
||||
self.last_frame = None
|
||||
self.frame_queue = asyncio.Queue(10)
|
||||
|
||||
async def put_frame(self, frame):
|
||||
await self.frame_queue.put(VideoFrame.from_ndarray(frame))
|
||||
|
||||
async def recv(self):
|
||||
if self.last_frame is None:
|
||||
frame = await self.frame_queue.get()
|
||||
|
||||
else:
|
||||
try:
|
||||
frame = self.frame_queue.get_nowait()
|
||||
|
||||
except asyncio.QueueEmpty:
|
||||
frame = self.last_frame
|
||||
|
||||
pts, time_base = await self.next_timestamp()
|
||||
|
||||
frame.pts = pts
|
||||
frame.time_base = time_base
|
||||
self.last_frame = frame
|
||||
return frame
|
||||
|
||||
|
||||
def _create_rectangle(width, height, color):
|
||||
data_bgr = numpy.zeros((height, width, 3), numpy.uint8)
|
||||
data_bgr[:, :] = color
|
||||
return data_bgr
|
||||
|
||||
|
||||
def _gen_test_image(k):
|
||||
height, width = 480, 640
|
||||
|
||||
# generate flag
|
||||
data_bgr = numpy.hstack(
|
||||
[
|
||||
_create_rectangle(width=213, height=480, color=(255, 0, 0)), # blue
|
||||
_create_rectangle(width=214, height=480, color=(255, 255, 255)), # white
|
||||
_create_rectangle(width=213, height=480, color=(0, 0, 255)), # red
|
||||
]
|
||||
)
|
||||
|
||||
# shrink and center it
|
||||
M = numpy.float32([[0.5, 0, width / 4], [0, 0.5, height / 4]])
|
||||
data_bgr = cv2.warpAffine(data_bgr, M, (width, height))
|
||||
|
||||
# compute animation
|
||||
omega = 2 * math.pi / height
|
||||
id_x = numpy.tile(numpy.array(range(width), dtype=numpy.float32), (height, 1))
|
||||
id_y = numpy.tile(numpy.array(range(height), dtype=numpy.float32), (width, 1)).transpose()
|
||||
|
||||
phase = 2 * k * math.pi / 30
|
||||
map_x = id_x + 10 * numpy.cos(omega * id_x + phase)
|
||||
map_y = id_y + 10 * numpy.sin(omega * id_x + phase)
|
||||
return cv2.remap(data_bgr, map_x, map_y, cv2.INTER_LINEAR)
|
||||
|
||||
|
||||
async def gen_image_frames(video_track):
|
||||
k = 0
|
||||
while True:
|
||||
if k > 500:
|
||||
k = 0
|
||||
img = _gen_test_image(k)
|
||||
k += 1
|
||||
|
||||
await video_track.put_frame(img)
|
||||
|
||||
|
||||
def force_codec(pc, sender, forced_codec):
|
||||
kind = forced_codec.split("/")[0]
|
||||
codecs = RTCRtpSender.getCapabilities(kind).codecs
|
||||
transceiver = next(t for t in pc.getTransceivers() if t.sender == sender)
|
||||
transceiver.setCodecPreferences([codec for codec in codecs if codec.mimeType == forced_codec])
|
||||
|
||||
|
||||
async def offer(request):
|
||||
params = await request.json()
|
||||
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
|
||||
|
||||
pc = RTCPeerConnection()
|
||||
pcs.add(pc)
|
||||
|
||||
@pc.on("connectionstatechange")
|
||||
async def on_connectionstatechange():
|
||||
print("Connection state is %s" % pc.connectionState)
|
||||
if pc.connectionState == "failed":
|
||||
await pc.close()
|
||||
pcs.discard(pc)
|
||||
|
||||
video = SingleFrameStreamTrack()
|
||||
loop = asyncio.get_event_loop()
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(gen_image_frames(video), loop)
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
pc.close()
|
||||
|
||||
if video:
|
||||
video_sender = pc.addTrack(video)
|
||||
if args.video_codec:
|
||||
force_codec(pc, video_sender, args.video_codec)
|
||||
elif args.play_without_decoding:
|
||||
raise Exception("You must specify the video codec using --video-codec")
|
||||
|
||||
await pc.setRemoteDescription(offer)
|
||||
|
||||
answer = await pc.createAnswer()
|
||||
await pc.setLocalDescription(answer)
|
||||
|
||||
return web.Response(
|
||||
content_type="application/json",
|
||||
text=json.dumps({"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}),
|
||||
)
|
||||
|
||||
|
||||
pcs = set()
|
||||
|
||||
|
||||
async def on_shutdown(app):
|
||||
# close peer connections
|
||||
coros = [pc.close() for pc in pcs]
|
||||
await asyncio.gather(*coros)
|
||||
pcs.clear()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="WebRTC webcam demo")
|
||||
parser.add_argument("--cert-file", help="SSL certificate file (for HTTPS)")
|
||||
parser.add_argument("--key-file", help="SSL key file (for HTTPS)")
|
||||
parser.add_argument(
|
||||
"--play-without-decoding",
|
||||
help=(
|
||||
"Read the media without decoding it (experimental). "
|
||||
"For now it only works with an MPEGTS container with only H.264 video."
|
||||
),
|
||||
action="store_true",
|
||||
)
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Host for HTTP server (default: 0.0.0.0)")
|
||||
parser.add_argument("--port", type=int, default=8052, help="Port for HTTP server (default: 8080)")
|
||||
parser.add_argument("--verbose", "-v", action="count")
|
||||
parser.add_argument("--audio-codec", help="Force a specific audio codec (e.g. audio/opus)")
|
||||
parser.add_argument("--video-codec", help="Force a specific video codec (e.g. video/H264)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbose:
|
||||
logging.basicConfig(level=logging.DEBUG)
|
||||
else:
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
|
||||
if args.cert_file:
|
||||
ssl_context = ssl.SSLContext()
|
||||
ssl_context.load_cert_chain(args.cert_file, args.key_file)
|
||||
else:
|
||||
ssl_context = None # pylint: disable=invalid-name
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_route("POST", "/offer", offer)
|
||||
app.on_shutdown.append(on_shutdown)
|
||||
cors = aiohttp_cors.setup(
|
||||
app,
|
||||
defaults={
|
||||
"*": aiohttp_cors.ResourceOptions(
|
||||
allow_credentials=True,
|
||||
expose_headers="*",
|
||||
allow_headers="*",
|
||||
)
|
||||
},
|
||||
)
|
||||
for route in list(app.router.routes()):
|
||||
cors.add(route)
|
||||
|
||||
web.run_app(app, host=args.host, port=args.port, ssl_context=ssl_context)
|
||||
@@ -217,3 +217,11 @@ def set_camera(vis, camera: Camera):
|
||||
pose = camera.get_camera_to_world_h()
|
||||
K = camera.get_intrinsics_matrix()
|
||||
set_persp_camera(vis, pose=pose.double().numpy(), K=K.double().numpy())
|
||||
|
||||
|
||||
def draw_aabb(vis, aabb, name="aabb"):
|
||||
"""Draw the axis-aligned bounding box."""
|
||||
lengths = aabb[1] - aabb[0]
|
||||
vis[name].set_object(g.Box(lengths.tolist()), material=g.MeshPhongMaterial(color=0xFF0000, opacity=0.1))
|
||||
center = aabb[0] + lengths / 2.0
|
||||
vis[name].set_transform(tf.translation_matrix(center.tolist()))
|
||||
@@ -14,14 +14,11 @@
|
||||
|
||||
from __future__ import absolute_import, division, print_function
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import umsgpack
|
||||
import zmq
|
||||
|
||||
from .commands import Delete, SetAnimation, SetImage, SetObject, SetProperty, SetTransform
|
||||
from .commands import Delete, SetObject, GetObject, SetImage, SetProperty, SetTransform
|
||||
from .path import Path
|
||||
|
||||
|
||||
@@ -42,19 +39,17 @@ class ViewerWindow(object):
|
||||
umsgpack.packb(cmd_data),
|
||||
]
|
||||
)
|
||||
self.client.recv()
|
||||
return self.client.recv()
|
||||
|
||||
|
||||
class Visualizer(object):
|
||||
__slots__ = ["window", "path"]
|
||||
|
||||
def __init__(self, window):
|
||||
self.window = window
|
||||
self.path = Path(("pyrad",)) # TODO(ethan): change this
|
||||
|
||||
@staticmethod
|
||||
def view_into(window, path):
|
||||
vis = Visualizer(window=window)
|
||||
vis = Visualizer(window)
|
||||
vis.path = path
|
||||
return vis
|
||||
|
||||
@@ -64,6 +59,12 @@ class Visualizer(object):
|
||||
def set_object(self, geometry, material=None):
|
||||
return self.window.send(SetObject(geometry, material, self.path))
|
||||
|
||||
def get_object(self):
|
||||
return self.window.send(GetObject(self.path))
|
||||
|
||||
def set_image(self, image):
|
||||
return self.window.send(SetImage(image, self.path))
|
||||
|
||||
def set_transform(self, matrix=np.eye(4)):
|
||||
assert matrix.shape == (4, 4)
|
||||
return self.window.send(SetTransform(matrix, self.path))
|
||||
@@ -71,12 +72,6 @@ class Visualizer(object):
|
||||
def set_property(self, key, value):
|
||||
return self.window.send(SetProperty(key, value, self.path))
|
||||
|
||||
def set_animation(self, animation, play=True, repetitions=1):
|
||||
return self.window.send(SetAnimation(animation, play=play, repetitions=repetitions))
|
||||
|
||||
def set_image(self, image):
|
||||
return self.window.send(SetImage(image, self.path))
|
||||
|
||||
def delete(self):
|
||||
return self.window.send(Delete(self.path))
|
||||
|
||||
|
||||
Reference in new issue
Block a user