Integrating WebRTC into viewer (#73)

* restructuring for webrtc integration

* some webrtc code but not working

* webrtc connects when muted

* the flag is real!

* updating camera in websocket bridge. moved webrtc to existing websocket connection instead of http request. need to change video track

* rudimentary adding real-time imaging

* added free TURN server so code works everywhere

* slow rendering works during training with the viewer

* fix tensor device and update default model save dir

* forgot to include trainer updates

* removing some code that isn't needed

* quick fix to bins. not the most accurate however

* cleaned up code and added some config logic

* black

* adding license

Co-authored-by: evonneng <evonne_ng@berkeley.edu>
This commit is contained in:
Ethan Weberandevonneng authored and GitHub committed 2022-07-05 14:52:49 -07:00
1 parent e507c97e3e
commit bf13ed3cf3
28 files changed
+8923 -917

No files matched your search

+7
View File
@@ -98,3 +98,10 @@ optimizers:
_target_: pyrad.optimizers.optimizers.ExponentialDecaySchedule
lr_final: 0.000005
max_steps: 1000000
viewer:
enable: False
zmq_url: tcp://127.0.0.1:6000
render_image_height: 100
steps_per_render_image: 100
num_rays_per_chunk: 4096
+6
View File
@@ -23,3 +23,9 @@ optimizers:
optimizer:
lr: 1e-2
scheduler: null
viewer:
enable: True
render_image_height: 200
steps_per_render_image: 5
num_rays_per_chunk: 16384
+3
View File
@@ -1,3 +1,6 @@
aiohttp
aiortc
av
black[jupyter]==22.3.0
functorch==0.1.1
h5py>=2.9.0
+119 -46
View File
@@ -10,7 +10,20 @@
},
{
"cell_type": "code",
"execution_count": 1,
"execution_count": null,
"id": "8501e01d-f932-4a75-8259-2def67a546d7",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"\n",
"os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
"os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"4\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b15dfdc1-5054-40bc-9631-4c7426eecfd9",
"metadata": {},
"outputs": [],
@@ -35,6 +48,8 @@
"from pyrad.utils.plotly import get_line_segments_from_lines\n",
"from pyrad.cameras.cameras import get_camera_model\n",
"from pyrad.utils.misc import get_dict_to_torch, instantiate_from_dict_config\n",
"from pyrad.engine.trainer import Trainer\n",
"from pyrad.utils.config import Config, setup_config\n",
"\n",
"# the new import\n",
"from pyrad.viewer.backend import vis_utils\n",
@@ -44,19 +59,31 @@
"from omegaconf import open_dict\n",
"import pprint\n",
"from tqdm import tqdm\n",
"import random"
"import random\n",
"\n",
"import umsgpack\n",
"from pyrad.viewer.backend.utils import get_chunks, get_intrinsics_matrix_and_camera_to_world_h"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "df82e083-e261-43e0-a1dc-05c6a59d16e1",
"execution_count": null,
"id": "032e4c9e-25be-4cc0-9ac8-a5be723b775a",
"metadata": {},
"outputs": [],
"source": [
"# establish the connection to the tcp backend, to talk to the visualizer\n",
"# zmq_url should match the output of `python run_zmq_server.py`\n",
"vis = vis_utils.get_vis(zmq_url=\"tcp://127.0.0.1:6000\")\n",
"vis = vis_utils.get_vis(zmq_url=\"tcp://127.0.0.1:6000\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "df82e083-e261-43e0-a1dc-05c6a59d16e1",
"metadata": {},
"outputs": [],
"source": [
"vis.delete()\n",
"\n",
"# draws a red box in the scene\n",
@@ -65,18 +92,99 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"id": "7fe7478d",
"metadata": {},
"outputs": [],
"source": [
"with initialize(version_base=None, config_path=\"../configs\"):\n",
" config = compose(config_name=\"graph_default.yaml\")"
" config = compose(\n",
" config_name=\"graph_default.yaml\",\n",
" overrides=[\n",
" \"trainer.resume_train.load_dir=/shared/ethanweber/pyrad/outputs/blender_lego/vanilla_nerf/2022-07-04_221907/checkpoints\",\n",
" \"trainer.resume_train.load_step=4000\",\n",
" ],\n",
" )\n",
" config = setup_config(config)"
]
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"id": "5d231e36-dd38-489b-a3a3-860cdf49e551",
"metadata": {},
"outputs": [],
"source": [
"trainer = Trainer(config, local_rank=0, world_size=1)\n",
"trainer.setup()\n",
"trainer.draw_scene_in_viewer()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1aba9ecb-0022-4896-833d-577abe532d23",
"metadata": {},
"outputs": [],
"source": [
"aabb = trainer.dataset_inputs_train.scene_bounds.aabb\n",
"vis_utils.draw_aabb(trainer.vis, aabb, name=\"dataset_inputs_train/scene_bounds/aabb\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4b23fc89-bfc3-40c4-abba-fad1317aeb21",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "5a829d31-4fc6-4908-963d-7326d3aa52da",
"metadata": {},
"outputs": [],
"source": [
"outputs = trainer.render_image_in_viewer()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "84bd64de-2e0c-458e-a4b7-1084f1cca1c5",
"metadata": {},
"outputs": [],
"source": [
"trainer.vis.delete()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6ed9cdfa-6359-4a70-95c7-033e853e9b50",
"metadata": {},
"outputs": [],
"source": [
"while True:\n",
" outputs = trainer.render_image_in_viewer()\n",
" image = outputs[\"rgb_fine\"].cpu().numpy().astype(\"float64\") * 255\n",
" trainer.vis[\"/Cameras/Main Camera\"].set_image(image)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b3bcd221-05a8-494e-b817-3e2de7f5a0df",
"metadata": {},
"outputs": [],
"source": [
"image.shape"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "642b0a25-0e88-4eac-b661-a57a74fb10fc",
"metadata": {},
"outputs": [],
@@ -86,7 +194,7 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": null,
"id": "938f090b-3b28-4fd6-82dc-d4b5c5917134",
"metadata": {},
"outputs": [],
@@ -97,45 +205,10 @@
},
{
"cell_type": "code",
"execution_count": 6,
"id": "5d014a6b-50fa-48ab-b135-0ab2b81c7529",
"metadata": {},
"outputs": [],
"source": [
"# grab 10 images and show their frustums\n",
"indices = random.sample(range(len(image_dataset_train)), k=10)\n",
"for idx in indices:\n",
" image = image_dataset_train[idx][\"image\"]\n",
" camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None)\n",
" pose = camera.get_camera_to_world().double().numpy()\n",
" K = camera.get_intrinsics_matrix().double().numpy()\n",
" vis_utils.draw_camera_frustum(\n",
" vis,\n",
" image=(image.double().numpy() * 255.0),\n",
" pose=pose,\n",
" K=K,\n",
" height=1.0,\n",
" name=\"{:06d}\".format(idx),\n",
" displayed_focal_length=0.5,\n",
" realistic=False,\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": null,
"id": "fd04703a-aa50-45e4-95fd-4d881ff18d61",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"estimated_time: 3.3333333333333335\n",
"time_elapsed: 3.5769083499908447\n"
]
}
],
"outputs": [],
"source": [
"# move a camera around\n",
"num_cameras = len(dataset_inputs.intrinsics)\n",
+88 -8
View File
@@ -17,13 +17,16 @@ Code to train model.
"""
import logging
import os
import random
from typing import Dict
import torch
import torch.distributed as dist
import umsgpack
from torch.nn.parallel import DistributedDataParallel as DDP
from torchtyping import TensorType
from pyrad.cameras.cameras import get_camera, get_intrinsics_from_intrinsics_matrix
from pyrad.cameras.rays import RayBundle
from pyrad.data.dataloader import EvalDataloader, setup_dataset_eval, setup_dataset_train
from pyrad.graphs.base import setup_graph
@@ -32,6 +35,9 @@ from pyrad.utils import profiler, writer
from pyrad.utils.config import Config
from pyrad.utils.decorators import check_main_thread
from pyrad.utils.writer import EventName, TimeWriter
from pyrad.viewer.backend import vis_utils
from pyrad.viewer.backend.utils import get_intrinsics_matrix_and_camera_to_world_h
from pyrad.viewer.backend.visualizer import ViewerWindow, Visualizer
logging.getLogger("PIL").setLevel(logging.WARNING)
@@ -40,10 +46,9 @@ class Trainer:
"""Training class
Args:
config (Config): _description_
local_rank (int, optional): _description_. Defaults to 0.
world_size (int, optional): _description_. Defaults to 1.
cpu (bool, optional): Whether or not to use the CPU.
config (Config): The configuration object.
local_rank (int, optional): Local rank of the process. Defaults to 0.
world_size (int, optional): World size of the process. Defaults to 1.
"""
def __init__(self, config: Config, local_rank: int = 0, world_size: int = 1):
@@ -52,6 +57,7 @@ class Trainer:
self.world_size = world_size
self.device = "cpu" if world_size == 0 else f"cuda:{local_rank}"
# dataset variables
self.dataset_inputs_train = None
self.dataloader_train = None
self.dataloader_eval = None
# model variables
@@ -62,11 +68,20 @@ class Trainer:
writer.setup_event_writers(config.logging, max_iter=config.trainer.max_num_iterations)
profiler.setup_profiler(config.logging)
self.vis = None
if self.config.viewer.enable:
window = ViewerWindow(zmq_url=self.config.viewer.zmq_url)
self.vis = Visualizer(window=window)
logging.info("Connected to viewer at %s", self.config.viewer.zmq_url)
self.vis.delete()
else:
logging.info("Continuing without viewer.")
def setup(self, test_mode=False):
"""Setup the Trainer by calling other setup functions."""
dataset_inputs_train, self.dataloader_train = setup_dataset_train(self.config.data, device=self.device)
self.dataset_inputs_train, self.dataloader_train = setup_dataset_train(self.config.data, device=self.device)
_, self.dataloader_eval = setup_dataset_eval(self.config.data, test_mode=test_mode, device=self.device)
self.graph = setup_graph(self.config.graph, dataset_inputs_train, device=self.device)
self.graph = setup_graph(self.config.graph, self.dataset_inputs_train, device=self.device)
self.optimizers = setup_optimizers(self.config.optimizers, self.graph.get_param_groups())
if self.config.trainer.resume_train.load_dir:
@@ -90,7 +105,11 @@ class Trainer:
return loss_sum
def train(self) -> None:
"""_summary_"""
"""Train the model."""
if self.vis:
self.draw_scene_in_viewer()
with TimeWriter(writer, EventName.TOTAL_TRAIN_TIME):
num_iterations = self.config.trainer.max_num_iterations
iter_dataloader_train = iter(self.dataloader_train)
@@ -106,6 +125,13 @@ class Trainer:
writer.put_dict(name="Loss/train-loss_dict", scalar_dict=loss_dict, step=step)
if step != 0 and self.config.trainer.steps_per_save and step % self.config.trainer.steps_per_save == 0:
self._save_checkpoint(self.config.trainer.model_dir, step)
if (
self.vis
and step != 0
and self.config.viewer.steps_per_render_image
and step % self.config.viewer.steps_per_render_image == 0
):
_ = self.render_image_in_viewer()
if step % self.config.trainer.steps_per_test == 0:
self.eval_with_dataloader(self.dataloader_eval, step=step)
self._write_out_storage(step)
@@ -128,7 +154,7 @@ class Trainer:
def _load_checkpoint(self) -> None:
"""helper function to load graph and optimizer from prespecified checkpoint"""
load_config = self.config.resume_train
load_config = self.config.trainer.resume_train
load_path = os.path.join(load_config.load_dir, f"step-{load_config.load_step:09d}.ckpt")
assert os.path.exists(load_path), f"Checkpoint {load_path} does not exist"
loaded_state = torch.load(load_path, map_location="cpu")
@@ -200,6 +226,60 @@ class Trainer:
self.graph.train()
return psnr
def draw_scene_in_viewer(self):
"""Draw some images and the scene aabb in the viewer."""
image_dataset_train = self.dataloader_train.image_sampler.dataset
dataset_inputs = self.dataset_inputs_train
indices = random.sample(range(len(image_dataset_train)), k=10)
for idx in indices:
image = image_dataset_train[idx]["image"]
camera = get_camera(dataset_inputs.intrinsics[idx], dataset_inputs.camera_to_world[idx], None)
pose = camera.get_camera_to_world().double().numpy()
K = camera.get_intrinsics_matrix().double().numpy()
vis_utils.draw_camera_frustum(
self.vis,
image=(image.double().numpy() * 255.0),
pose=pose,
K=K,
height=1.0,
name=f"image_dataset_train/{idx:06d}",
displayed_focal_length=0.5,
realistic=False,
)
aabb = dataset_inputs.scene_bounds.aabb
vis_utils.draw_aabb(self.vis, aabb, name="dataset_inputs_train/scene_bounds/aabb")
@profiler.time_function
def render_image_in_viewer(self):
"""
Draw an image using the current camera pose from the viewer.
The image is sent of a TCP connection and then uses WebRTC to send it to the viewer.
"""
data = self.vis["/Cameras/Main Camera"].get_object()
message = umsgpack.unpackb(data)
camera_object = message["object"]["object"]
image_height = self.config.viewer.render_image_height
intrinsics_matrix, camera_to_world_h = get_intrinsics_matrix_and_camera_to_world_h(
camera_object, image_height=image_height
)
camera_to_world = camera_to_world_h[:3, :]
intrinsics = get_intrinsics_from_intrinsics_matrix(intrinsics_matrix)
camera = get_camera(intrinsics, camera_to_world)
camera_ray_bundle = camera.get_camera_ray_bundle(device=self.device)
camera_ray_bundle.num_rays_per_chunk = self.config.viewer.num_rays_per_chunk
self.graph.eval()
outputs = self.graph.get_outputs_for_camera_ray_bundle(camera_ray_bundle)
self.graph.train()
# gross hack to get the image key, depending on which keys the graph uses
rgb_key = "rgb" if "rgb" in outputs else "rgb_fine"
# TODO: make it such that the TCP connection doesn't need float64
image = outputs[rgb_key].cpu().numpy().astype("float64") * 255
self.vis["/Cameras/Main Camera"].set_image(image)
return outputs
def eval_with_dataloader(self, dataloader: EvalDataloader, step: int = None) -> None:
"""Run evaluation with a given dataloader.
+3
View File
@@ -116,6 +116,7 @@ class Config:
data: DataConfig = MISSING
graph: GraphConfig = MISSING
optimizers: Dict[str, Any] = MISSING
viewer: Dict[str, Any] = MISSING
# additional optional parameters here
hydra: Optional[Dict[str, Any]] = None
@@ -137,6 +138,7 @@ def setup_config(config: DictConfig) -> Config:
data = DataConfig(**config.data)
graph = GraphConfig(**config.graph)
optimizers = config.optimizers
viewer = config.viewer
return Config(
machine=machine,
logging=logging,
@@ -146,4 +148,5 @@ def setup_config(config: DictConfig) -> Config:
data=data,
graph=graph,
optimizers=optimizers,
viewer=viewer,
)
+10 -10
View File
@@ -1,14 +1,12 @@
<!DOCTYPE html>
<html lang="en">
<head>
<head>
<meta charset="utf-8" />
<link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<meta name="theme-color" content="#000000" />
<meta
name="description"
content="Web site created using create-react-app"
/>
<meta name="description" content="Web site created using create-react-app" />
<link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
<!--
manifest.json provides metadata used when your web app is installed on a
@@ -24,9 +22,10 @@
work correctly both with client-side routing and a non-root public URL.
Learn how to configure a non-root public URL by running `npm run build`.
-->
<title>visualizer</title>
</head>
<body>
<title>PyRad Viewer</title>
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>
<!--
@@ -39,5 +38,6 @@
To begin the development, run `npm start` or `yarn start`.
To create a production bundle, use `npm run build` or `yarn build`.
-->
</body>
</html>
</body>
</html>
-38
View File
@@ -1,38 +0,0 @@
.App {
/*text-align: center;*/
}
.App-logo {
height: 40vmin;
pointer-events: none;
}
@media (prefers-reduced-motion: no-preference) {
.App-logo {
animation: App-logo-spin infinite 20s linear;
}
}
.App-header {
background-color: #282c34;
min-height: 100vh;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
font-size: calc(10px + 2vmin);
color: white;
}
.App-link {
color: #61dafb;
}
@keyframes App-logo-spin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
-45
View File
@@ -1,45 +0,0 @@
import React, { Component } from "react";
import "./App.css";
import { Viewer } from "./Viewer";
class Image extends Component {
constructor(props) {
super(props);
this.state = {};
}
}
class ColorGUIHelper {
constructor(object, prop) {
this.object = object;
this.prop = prop;
}
get value() {
return `#${this.object[this.prop].getHexString()}`;
}
set value(hexString) {
this.object[this.prop].set(hexString);
}
}
class App extends Component {
constructor(props) {
super(props);
this.state = {
client: null,
};
this.viewer_ref = React.createRef();
}
render() {
return (
<div className="App">
<Viewer ref={this.viewer_ref} />
</div>
);
}
}
export default App;
-9
View File
@@ -1,9 +0,0 @@
import React from 'react';
import { render } from '@testing-library/react';
import App from './App';
test('renders learn react link', () => {
const { getByText } = render(<App />);
const linkElement = getByText(/learn react/i);
expect(linkElement).toBeInTheDocument();
});
+19
View File
@@ -0,0 +1,19 @@
import React, { Component } from "react";
export class BackgroundDiv extends Component {
componentDidMount() {
document.getElementById("BackgroundDiv-video").play();
}
render() {
return (
<div className="BackgroundDiv">
<video id="BackgroundDiv-video" autoPlay muted loop>
<source
src="https://www.w3schools.com/html/mov_bbb.mp4"
type="video/mp4"
></source>
</video>{" "}
</div>
);
}
}
+2 -22
View File
@@ -36,11 +36,10 @@ function dispose(object) {
}
export class SceneNode {
constructor(object, folder, on_update) {
constructor(object, folder) {
this.object = object;
this.folder = folder;
this.children = {};
this.on_update = on_update;
this.create_controls();
for (let c of this.object.children) {
this.add_child(c);
@@ -49,7 +48,7 @@ export class SceneNode {
add_child(object) {
let f = this.folder.addFolder(object.name);
let node = new SceneNode(object, f, this.on_update);
let node = new SceneNode(object, f);
this.children[object.name] = node;
return node;
}
@@ -79,28 +78,9 @@ export class SceneNode {
this.folder.domElement.removeChild(this.vis_controller.domElement);
}
this.vis_controller = new dat.controllers.BooleanController(this.object, "visible");
this.vis_controller.onChange(() => this.on_update());
this.folder.domElement.prepend(this.vis_controller.domElement);
this.vis_controller.domElement.style.height = "0";
this.vis_controller.domElement.style.float = "right";
// // add on hover controls and clickable functionality
// // hover
// this.folder.domElement.addEventListener("mouseenter", function (event) {
// event.target.style.color = "purple";
// }, false);
// this.folder.domElement.addEventListener("mouseleave", function (event) {
// event.target.style.color = "";
// }, false);
// // click
// this.folder.domElement.addEventListener("dblclick", () => this.printValue(), false);
}
printValue() {
console.log("calling print value");
console.log(this.object);
console.log(this);
}
set_property(property, value) {
+165 -332
View File
@@ -1,5 +1,4 @@
import React, { Component } from "react";
import "./App.css";
import * as THREE from "three";
import { TrackballControls } from "three/examples/jsm/controls/TrackballControls";
import Stats from "three/examples/jsm/libs/stats.module.js";
@@ -12,26 +11,17 @@ function websocket_endpoint_from_url(url) {
let endpoint = url.split("/").pop();
if (endpoint == "") {
let message =
"Please set a websocket endpoint. For example, a correct URL may look like the following: http://localhost:3000/visiongpu10.csail.mit.edu:8051";
"Please set the websocket endpoint. E.g., a correct URL may be: http://localhost:4000/localhost:8051";
window.alert(message);
return null;
}
return endpoint;
}
function removeAllChildNodes(parent) {
// https://www.javascripttutorial.net/dom/manipulating/remove-all-child-nodes/
while (parent.firstChild) {
parent.removeChild(parent.firstChild);
}
}
function findCameraObjectUnderObject3D(object) {
// seach the tree for the camera
if (object instanceof THREE.Camera) {
return object;
} else if (object instanceof THREE.CameraHelper) {
return object.camera;
}
for (let name of Object.keys(object.children)) {
return findCameraObjectUnderObject3D(object.children[name]);
@@ -50,87 +40,51 @@ export class Viewer extends Component {
renderer: null,
gui: null,
scene_tree: null,
needs_render: true,
viewport_width: null,
viewport_height: null,
images: [],
image_index: 0,
};
this.animate = this.animate.bind(this);
this.update = this.update.bind(this);
this.set_object = this.set_object.bind(this);
this.handle_command = this.handle_command.bind(this);
// window.addEventListener('resize', this.handleResize.bind(this));
}
set_dirty() {
this.setState({ needs_render: true });
this.threejs_ref = React.createRef();
}
handleResize() {
this.state.viewport_width = this.getViewportWidth();
this.state.viewport_height = this.getViewportHeight();
this.state.camera_l.aspect =
this.state.camera_main.aspect =
this.state.viewport_width / this.state.viewport_height;
this.state.camera_l.updateProjectionMatrix();
this.state.camera_r.updateProjectionMatrix();
this.state.camera_orth.updateProjectionMatrix();
this.state.renderer_l.setSize(
this.state.camera_main.updateProjectionMatrix();
this.state.renderer_main.setSize(
this.state.viewport_width,
(this.state.viewport_height * 2) / 3
);
this.state.renderer_r.setSize(
this.state.viewport_width / 2,
(this.state.viewport_height * 1) / 3
);
this.state.renderer_orth.setSize(
this.state.viewport_width / 2,
(this.state.viewport_height * 1) / 3
this.state.viewport_height
);
}
animate() {
requestAnimationFrame(this.animate);
if (this.state.needs_render) {
this.handleResize();
let camera_l = findCameraObjectUnderObject3D(
this.state.scene_tree.find(["Cameras", "Main Camera L"]).object
);
let camera_r = findCameraObjectUnderObject3D(
this.state.scene_tree.find(["Cameras", "Main Camera R"]).object
);
let camera_orth = findCameraObjectUnderObject3D(
this.state.scene_tree.find(["Cameras", "Main Camera Orth"]).object
);
camera_l.updateProjectionMatrix();
camera_r.updateProjectionMatrix();
camera_orth.updateProjectionMatrix();
if (this.state.controls_l != null) {
this.state.controls_l.update();
}
if (this.state.controls_r != null) {
this.state.controls_r.update();
}
if (this.state.controls_orth != null) {
this.state.controls_orth.update();
}
this.state.renderer_l.render(this.state.scene, camera_l);
this.state.renderer_r.render(this.state.scene, camera_r);
// NOTE(ethan): trying to add background image
// console.log("ethan is here");
if (this.state.websocket_is_opened) {
this.state.websocket.send("TODO: put camera pose here");
}
this.state.renderer_orth.render(
this.backgroundScene,
this.backgroundCamera
);
// this.state.renderer_orth.render(this.state.scene, camera_orth);
this.state.needs_render = false;
this.set_dirty(); // TODO(ethan): remove this but make sure to include it everywhere
update() {
requestAnimationFrame(this.update);
this.handleResize();
let camera_main = findCameraObjectUnderObject3D(
this.state.scene_tree.find(["Cameras", "Main Camera"]).object
);
camera_main.updateProjectionMatrix();
if (this.state.controls_main != null) {
this.state.controls_main.update();
}
this.state.renderer_main.render(this.state.scene, camera_main);
if (this.state.websocket.readyState === WebSocket.OPEN) {
// update the camera information in the python server
let cmd = "set_object";
let path = "Cameras/Main Camera";
let data = {
type: cmd,
path: path,
object: camera_main.toJSON(),
};
let message = msgpack.encode(data);
this.state.websocket.send(message);
}
}
@@ -155,16 +109,25 @@ export class Viewer extends Component {
set_object(path, object) {
this.state.scene_tree.find(path.concat(["<object>"])).set_object(object);
// add controls if object is a camera
if (object instanceof THREE.Camera) {
this.state.controls_main = new TrackballControls(
object,
this.state.renderer_main.domElement
);
this.state.controls_main.rotateSpeed = 2.0;
this.state.controls_main.zoomSpeed = 0.3;
this.state.controls_main.panSpeed = 0.2;
this.state.controls_main.staticMoving = false; // false is default
this.state.controls_main.target.set(0, 0, 0); // focus point of the controls
this.state.controls_main.autoRotate = false;
this.state.controls_main.update();
}
}
set_object_from_json(path, object_json) {
let loader = new ExtensibleObjectLoader();
loader.onTextureLoad = () => {
this.set_dirty();
};
loader.parse(object_json, (obj) => {
// console.log("ethan.");
// console.log(obj.type);
if (
obj.geometry !== undefined &&
obj.geometry.type === "BufferGeometry"
@@ -175,15 +138,10 @@ export class Viewer extends Component {
) {
obj.geometry.computeVertexNormals();
}
} else if (obj.type.includes("Camera")) {
console.log("uses camera !!");
// this.set_camera(obj);
// this.set_3d_pane_size();
}
obj.castShadow = true;
obj.receiveShadow = true;
this.set_object(path, obj);
this.set_dirty();
});
}
@@ -209,17 +167,11 @@ export class Viewer extends Component {
}
handle_command(cmd) {
console.log(cmd);
// convert binary serialization format back to JSON
cmd = msgpack.decode(new Uint8Array(cmd));
console.log(cmd);
// TODO(ethan): ignore these or remove status. maybe incorporate into a clean view
if (cmd.type === "status") {
return;
}
// three js scene commands
if (cmd.type === "set_object") {
let path = split_path(cmd.path);
this.set_object_from_json(path, cmd.object);
@@ -232,59 +184,14 @@ export class Viewer extends Component {
} else if (cmd.type === "set_property") {
let path = split_path(cmd.path);
this.set_property(path, cmd.property, cmd.value);
} else if (cmd.type === "set_animation") {
// TODO(ethan): implement animations
console.error("Animations not implemented yet.");
} else if (cmd.type === "set_image") {
console.log("ethan: setting camera");
// var url = URL.createObjectURL(
// new Blob([cmd["image"].buffer], { type: "image/png" } /* (1) */)
// );
// console.log(url);
// document.getElementById("my-img").src = url;
var height = 300;
var width = 200;
// console.log(cmd["image"]);
var texture = new THREE.DataTexture(
cmd["image"],
width,
height,
THREE.RGBFormat
);
// var texture = new THREE.TextureLoader().load("texture2.jpeg");
// var texture = new THREE.TextureLoader().load(url);
// console.log(texture);
// console.log(this.backgroundMesh);
this.backgroundMesh.material.map = texture;
this.backgroundMesh.material.map.needsUpdate = true;
this.backgroundMesh.material.needsUpdate = true;
}
this.set_dirty();
if (cmd.type === "set_transform" && cmd.path === "/Cameras/Main Camera R") {
let camera_r = findCameraObjectUnderObject3D(
this.state.scene_tree.find(["Cameras", "Main Camera R"]).object
);
camera_r.updateProjectionMatrix();
this.state.renderer_save.render(this.state.scene, camera_r);
let imgData = this.state.renderer_save.domElement.toDataURL();
this.state.images.push(imgData.toString());
// this.save_image(imgData);
// web rtc commands
else if (cmd.type === "answer") {
let answer = cmd.data;
this.state.pc.setRemoteDescription(answer);
}
}
save_scene() {}
load_scene() {}
// save_image() {}
add_camera_helper(path, camera) {
let camera_copy = camera.clone();
camera_copy.far = 1; // so it doesn't show so far away
this.set_object(path, new THREE.CameraHelper(camera_copy));
}
getViewportWidth() {
return window.innerWidth - (window.innerWidth % 2);
}
@@ -293,14 +200,94 @@ export class Viewer extends Component {
return window.innerHeight;
}
setupWebRTC() {
console.log("setting up WebRTC");
// The iceServers config comes from the following URL:
// https://www.metered.ca/tools/openrelay/
this.state.pc = new RTCPeerConnection({
iceServers: [
{
urls: "stun:openrelay.metered.ca:80",
},
{
urls: "turn:openrelay.metered.ca:80",
username: "openrelayproject",
credential: "openrelayproject",
},
{
urls: "turn:openrelay.metered.ca:443",
username: "openrelayproject",
credential: "openrelayproject",
},
{
urls: "turn:openrelay.metered.ca:443?transport=tcp",
username: "openrelayproject",
credential: "openrelayproject",
},
],
});
// connect video
this.state.pc.addEventListener("track", function (evt) {
if (evt.track.kind == "video") {
console.log("setting event stream");
document.getElementById("WebRTCVideo-video").srcObject = evt.streams[0];
}
});
this.state.pc.addTransceiver("video", { direction: "recvonly" });
this.state.pc
.createOffer()
.then((offer) => {
return this.state.pc.setLocalDescription(offer);
})
.then(() => {
// wait for ICE gathering to complete
return new Promise((resolve) => {
if (this.state.pc.iceGatheringState === "complete") {
resolve();
} else {
var checkState = () => {
if (this.state.pc.iceGatheringState === "complete") {
this.state.pc.removeEventListener(
"icegatheringstatechange",
checkState
);
resolve();
}
};
this.state.pc.addEventListener(
"icegatheringstatechange",
checkState
);
}
});
})
.then(() => {
var offer = this.state.pc.localDescription;
console.log("sending the offer");
let cmd = "offer";
let path = "";
let data = {
type: cmd,
path: path,
data: {
sdp: offer.sdp,
type: offer.type,
},
};
let message = msgpack.encode(data);
this.state.websocket.send(message);
});
}
componentDidMount() {
// Open the websocket
let endpoint = websocket_endpoint_from_url(window.location.href);
this.state.websocket_is_opened = false;
this.state.websocket = new WebSocket("ws://" + endpoint + "/");
this.state.websocket.onopen = () => {
console.log("connected");
this.state.websocket_is_opened = true;
console.log("websocket connected");
this.setupWebRTC();
};
this.state.websocket.binaryType = "arraybuffer";
// 'command' for updates to Viewer
@@ -310,195 +297,50 @@ export class Viewer extends Component {
};
// Get size of screen
this.state.viewport_width = this.getViewportWidth(); // notice that this is half width
this.state.viewport_width = this.getViewportWidth();
this.state.viewport_height = this.getViewportHeight();
// Scene
this.state.scene = new THREE.Scene();
this.state.scene.background = new THREE.Color(0xffffff);
// GUI
this.state.gui = new GUI();
let scene_folder = this.state.gui.addFolder("Scene");
scene_folder.open();
this.state.scene_tree = new SceneNode(this.state.scene, scene_folder, () =>
this.set_dirty()
);
// let save_folder = this.state.gui.addFolder("Save / Load / Capture");
// save_folder.add(this, 'save_scene');
// save_folder.add(this, 'load_scene');
// save_folder.add(this, 'save_image');
// this.state.gui.open();
this.state.scene_tree = new SceneNode(this.state.scene, scene_folder);
// console.log(this.mount);
// Renderer Left
this.state.renderer_l = new THREE.WebGLRenderer({ antialias: true });
this.state.renderer_l.setPixelRatio(window.devicePixelRatio);
this.state.renderer_l.setSize(
this.state.viewport_width,
(this.state.viewport_height * 2) / 3
);
this.state.renderer_l.domElement.style.border = "1px solid black";
this.mount.appendChild(this.state.renderer_l.domElement);
// Renderer Right
let div = document.createElement("div");
this.state.renderer_r = new THREE.WebGLRenderer({ antialias: true });
this.state.renderer_r.setPixelRatio(window.devicePixelRatio);
this.state.renderer_r.setSize(
this.state.viewport_width / 2,
(this.state.viewport_height * 1) / 3
);
this.state.renderer_r.domElement.style.border = "1px solid black";
this.state.renderer_r.domElement.style = "inline-block";
div.appendChild(this.state.renderer_r.domElement);
this.state.renderer_orth = new THREE.WebGLRenderer({ antialias: true });
this.state.renderer_orth.setPixelRatio(window.devicePixelRatio);
this.state.renderer_orth.setSize(
this.state.viewport_width / 2,
(this.state.viewport_height * 1) / 3
);
this.state.renderer_orth.domElement.style.border = "1px solid black";
this.state.renderer_orth.domElement.style = "inline-block";
div.appendChild(this.state.renderer_orth.domElement);
// div.display.style = "relative";
// NOTE(ethan): not sure if this is needed anymore?
this.state.renderer_save = new THREE.WebGLRenderer({
// Renderer main
this.state.renderer_main = new THREE.WebGLRenderer({
antialias: true,
preserveDrawingBuffer: true,
alpha: true,
});
this.state.renderer_save.setPixelRatio(1.0);
this.state.renderer_save.setSize(960, 540);
this.state.renderer_save.domElement.style.border = "1px solid black";
this.state.renderer_save.domElement.style.display = "none";
// div.appendChild(this.state.renderer_save.domElement);
this.state.renderer_main.setPixelRatio(window.devicePixelRatio);
this.state.renderer_main.setSize(
this.state.viewport_width,
this.state.viewport_height
);
this.state.renderer_main.domElement.style.border = "1px solid black";
this.mount.appendChild(div);
let domElement = document.getElementById("canvas-container-main");
domElement.append(this.state.renderer_main.domElement);
// console.log(this.mount);
this.mount.style.display = "block";
// Camera settings at https://stackoverflow.com/questions/46182845/field-of-view-aspect-ratio-view-matrix-from-projection-matrix-hmd-ost-calib/46195462
// Camera left
this.state.camera_l = new THREE.PerspectiveCamera(
// Camera main
this.state.camera_main = new THREE.PerspectiveCamera(
120,
this.state.viewport_width / this.state.viewport_height,
0.01,
100
);
this.state.camera_l.position.x = 5;
this.state.camera_l.position.y = -5;
this.state.camera_l.position.z = -5;
this.state.camera_l.up = new THREE.Vector3(0, -1, 0);
// Controls left
this.state.controls_l = new TrackballControls(
this.state.camera_l,
this.state.renderer_l.domElement
);
this.state.controls_l.rotateSpeed = 2.0;
this.state.controls_l.zoomSpeed = 0.3;
this.state.controls_l.panSpeed = 0.2;
this.state.controls_l.staticMoving = false; // false is default
this.state.controls_l.target.set(0, 0, 0); // focus point of the controls
this.state.controls_l.autoRotate = false;
this.state.controls_l.update();
// Camera right
this.state.camera_r = new THREE.PerspectiveCamera(
120,
this.state.viewport_width / (this.state.viewport_height / 2),
0.01,
100
);
this.state.camera_orth = new THREE.OrthographicCamera(
-2,
2,
-2,
2,
0.01,
100
);
// this.state.camera_r.position.x = 0;
// this.state.camera_r.position.y = 0;
// this.state.camera_r.position.z = 0;
// this.state.camera_r.up = new THREE.Vector3(0, -1, 0);
var texture = new THREE.TextureLoader().load("texture.jpeg");
this.backgroundMesh = new THREE.Mesh(
new THREE.PlaneGeometry(2, 2),
new THREE.MeshBasicMaterial({
map: texture,
})
// material
);
// const geometry = new THREE.PlaneGeometry(1, 1);
// const material = new THREE.MeshBasicMaterial({
// color: 0xffff00,
// side: THREE.DoubleSide,
// });
// const plane = new THREE.Mesh(geometry, material);
this.backgroundMesh.material.depthTest = false;
this.backgroundMesh.material.depthWrite = false;
// Create your background scene
this.backgroundScene = new THREE.Scene();
this.backgroundCamera = new THREE.Camera();
this.backgroundScene.add(this.backgroundCamera);
this.backgroundScene.add(this.backgroundMesh);
this.set_object(["Cameras", "Main Camera L"], this.state.camera_l);
this.set_object(["Cameras", "Main Camera R"], this.state.camera_r);
this.set_object(["Cameras", "Main Camera Orth"], this.state.camera_orth);
// rotate the cameras to match the COLMAP format
let rotY = new THREE.Matrix4();
rotY.makeRotationY(Math.PI);
let rotZ = new THREE.Matrix4();
rotZ.makeRotationZ(Math.PI);
let rot = rotY.multiply(rotZ);
this.set_transform(["Cameras", "Main Camera R", "<object>"], rot.toArray());
// this.state.scene_tree.find(["Cameras", "Main Camera L"]).set_property("position", [5, -5, -5]);
this.state.scene_tree
.find(["Cameras", "Main Camera R"])
.set_property("position", [-2, -2, -2]);
// Controls right
// this.state.controls_r = new TrackballControls(this.state.scene_tree.find(["Cameras", "Main Camera R"]).object, this.state.renderer_r.domElement);
// this.state.controls_r.rotateSpeed = 2.0;
// this.state.controls_r.zoomSpeed = 0.3;
// this.state.controls_r.panSpeed = 0.2;
// this.state.controls_r.staticMoving = false; // false is default
// this.state.controls_r.target.set(0, 0, 0); // focus point of the controls
// this.state.controls_r.autoRotate = false;
// this.state.controls_r.update();
this.set_transform(
["Cameras", "Main Camera Orth", "<object>"],
rot.toArray()
);
this.state.scene_tree
.find(["Cameras", "Main Camera Orth"])
.set_property("position", [-1, -1, -1]);
let camera_axes_l = new THREE.AxesHelper(1);
let camera_axes_r = new THREE.AxesHelper(1);
let camera_axes_orth = new THREE.AxesHelper(1);
this.set_object(["Cameras", "Main Camera L", "Axes"], camera_axes_l);
this.set_object(["Cameras", "Main Camera R", "Axes"], camera_axes_r);
this.set_object(["Cameras", "Main Camera Orth", "Axes"], camera_axes_orth);
this.state.camera_main.position.x = 5;
this.state.camera_main.position.y = -5;
this.state.camera_main.position.z = -5;
this.state.camera_main.up = new THREE.Vector3(0, -1, 0);
this.set_object(["Cameras", "Main Camera"], this.state.camera_main);
// Axes display
let axes = new THREE.AxesHelper(5);
this.set_object(["Axes"], axes);
// var arrowPos = new THREE.Vector3(0, 0, 0);
// var arrow = new THREE.ArrowHelper(new THREE.Vector3(1, 0, 0), arrowPos, 60, 0xFF0000, 20, 10)
// this.set_object(["arrow"], arrow);
// Grid
let grid = new THREE.GridHelper(20, 20);
this.set_object(["Grid"], grid);
@@ -511,32 +353,23 @@ export class Viewer extends Component {
// Stats
var stats = new Stats();
this.mount.appendChild(stats.dom);
let domElement_stats = document.getElementById("canvas-container-main");
domElement_stats.append(stats.dom);
// let camera = new THREE.PerspectiveCamera(75, this.state.width / this.state.height, 0.1, 1000);
// let helper = new THREE.CameraHelper(camera);
// // scene.add( helper );
// // console.log(helper);
// console.log(helper);
// // this.set_object(["Helper"], helper);
// // console.log(this.state.scene);
// Cube
// var geometry = new THREE.BoxGeometry(1, 1, 1);
// var material = new THREE.MeshBasicMaterial({color: 0x00ff00});
// var cube = new THREE.Mesh(geometry, material);
// this.state.scene.add(cube);
this.animate();
}
componentWillUnmount() {
// this.mount.removeChild(this.state.renderer_l.domElement);
// this.mount.removeChild(this.state.renderer_r.domElement);
removeAllChildNodes(this.mount);
this.update();
}
render() {
return <div ref={(ref) => (this.mount = ref)} />;
return (
<div>
<div className="WebRTCVideo">
<video id="WebRTCVideo-video" autoPlay playsInline muted>
{" "}
</video>{" "}
</div>{" "}
<div id="canvas-container-main"> </div>{" "}
<div id="stats-container"> </div>{" "}
</div>
);
}
}
+33 -3
View File
@@ -1,13 +1,43 @@
body {
margin: 0;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", "Oxygen",
"Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue",
sans-serif;
-webkit-font-smoothing: antialiased;
-moz-osx-font-smoothing: grayscale;
}
code {
font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
font-family: source-code-pro, Menlo, Monaco, Consolas, "Courier New",
monospace;
}
.BackgroundDiv,
.WebRTCVideo {
position: absolute;
top: 0px;
right: 0px;
bottom: 0px;
left: 0px;
}
#BackgroundDiv-video,
#WebRTCVideo-video {
width: 100%;
height: 100%;
object-fit: fill;
}
#canvas-container-main {
position: absolute;
top: 0px;
right: 0px;
bottom: 0px;
left: 0px;
}
#canvas-container-main canvas {
position: relative;
height: 100%;
width: 100%;
}
+6 -10
View File
@@ -1,17 +1,13 @@
import React from 'react';
import React from "react";
import ReactDOM from "react-dom/client";
import './index.css';
import App from './App';
import * as serviceWorker from './serviceWorker';
import "./index.css";
import * as serviceWorker from "./serviceWorker";
import { Viewer } from "./Viewer";
const root = ReactDOM.createRoot(document.getElementById("root"));
root.render(
<React.StrictMode>
<App />
</React.StrictMode>
);
root.render( < Viewer / > );
// If you want your app to work offline and load faster, you can change
// unregister() to register() below. Note this comes with some pitfalls.
// Learn more about service workers: https://bit.ly/CRA-PWA
serviceWorker.unregister();
serviceWorker.unregister();
File diff suppressed because it is too large. Load diff
+48
View File
@@ -0,0 +1,48 @@
function createPeerConnection() {
var config = {
sdpSemantics: "unified-plan",
};
// if (document.getElementById('use-stun').checked) {
// config.iceServers = [{urls: ['stun:stun.l.google.com:19302']}];
// }
let pc = new RTCPeerConnection(config);
// register some listeners to help debugging
pc.addEventListener(
"icegatheringstatechange",
function () {
iceGatheringLog.textContent += " -> " + pc.iceGatheringState;
},
false
);
iceGatheringLog.textContent = pc.iceGatheringState;
pc.addEventListener(
"iceconnectionstatechange",
function () {
iceConnectionLog.textContent += " -> " + pc.iceConnectionState;
},
false
);
iceConnectionLog.textContent = pc.iceConnectionState;
pc.addEventListener(
"signalingstatechange",
function () {
signalingLog.textContent += " -> " + pc.signalingState;
},
false
);
signalingLog.textContent = pc.signalingState;
// connect audio / video
pc.addEventListener("track", function (evt) {
if (evt.track.kind == "video")
document.getElementById("video").srcObject = evt.streams[0];
else document.getElementById("audio").srcObject = evt.streams[0];
});
return pc;
}
-1
View File
@@ -18,7 +18,6 @@ from . import commands
from . import geometry
from . import visualizer
from . import transformations
from . import animation
from .visualizer import ViewerWindow, Visualizer
-184
View File
@@ -1,184 +0,0 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division, print_function
import tempfile
import tarfile
import sys
import os.path
import subprocess
if sys.version_info >= (3, 0):
unicode = str
import bisect
from . import transformations as tf
class AnimationTrack(object):
__slots__ = ["name", "jstype", "frames", "values"]
def __init__(self, name, jstype, frames=None, values=None):
self.name = name
self.jstype = jstype
if frames is None:
self.frames = []
else:
self.frames = frames
if values is None:
self.values = []
else:
self.values = values
def set_property(self, frame, value):
i = bisect.bisect(self.frames, frame)
self.frames.insert(i, frame)
self.values.insert(i, value)
def lower(self):
return {
"name": unicode("." + self.name),
"type": unicode(self.jstype),
"keys": [{"time": self.frames[i], "value": self.values[i]} for i in range(len(self.frames))],
}
class AnimationClip(object):
__slots__ = ["tracks", "fps", "name"]
def __init__(self, tracks=None, fps=30, name="default"):
if tracks is None:
self.tracks = {}
else:
self.tracks = tracks
self.fps = fps
self.name = name
def set_property(self, frame, property, jstype, value):
if property not in self.tracks:
self.tracks[property] = AnimationTrack(property, jstype)
track = self.tracks[property]
track.set_property(frame, value)
def lower(self):
return {"fps": self.fps, "name": unicode(self.name), "tracks": [t.lower() for t in self.tracks.values()]}
class Animation(object):
__slots__ = ["clips", "default_framerate"]
def __init__(self, clips=None, default_framerate=30):
if clips is None:
self.clips = {}
else:
self.clips = clips
self.default_framerate = default_framerate
def lower(self):
return [{"path": path.lower(), "clip": clip.lower()} for (path, clip) in self.clips.items()]
def at_frame(self, visualizer, frame):
return AnimationFrameVisualizer(self, visualizer.path, frame)
def js_position(matrix):
return list(matrix[:3, 3])
def js_quaternion(matrix):
quat = tf.quaternion_from_matrix(matrix)
return [quat[1], quat[2], quat[3], quat[0]]
class AnimationFrameVisualizer(object):
__slots__ = ["animation", "path", "current_frame"]
def __init__(self, animation, path, current_frame):
self.animation = animation
self.path = path
self.current_frame = current_frame
def get_clip(self):
if self.path not in self.animation.clips:
self.animation.clips[self.path] = AnimationClip(fps=self.animation.default_framerate)
return self.animation.clips[self.path]
def set_transform(self, matrix):
assert matrix.shape == (4, 4)
clip = self.get_clip()
clip.set_property(self.current_frame, "position", "vector3", js_position(matrix))
clip.set_property(self.current_frame, "quaternion", "quaternion", js_quaternion(matrix))
def set_property(self, prop, jstype, value):
clip = self.get_clip()
clip.set_property(self.current_frame, prop, jstype, value)
def __getitem__(self, path):
return AnimationFrameVisualizer(self.animation, self.path.append(path), self.current_frame)
def __enter__(self):
return self
def __exit__(self, *arg):
pass
def convert_frames_to_video(tar_file_path, output_path="output.mp4", framerate=60, overwrite=False):
"""
Try to convert a tar file containing a sequence of frames saved by the
pyrad viewer into a single video file.
This relies on having `ffmpeg` installed on your system.
"""
output_path = os.path.abspath(output_path)
if os.path.isfile(output_path) and not overwrite:
raise ValueError(
"The output path {:s} already exists. To overwrite that file, you can pass overwrite=True to this function.".format(
output_path
)
)
with tempfile.TemporaryDirectory() as tmp_dir:
with tarfile.open(tar_file_path) as tar:
tar.extractall(tmp_dir)
args = [
"ffmpeg",
"-r",
str(framerate),
"-i",
r"%07d.png",
"-vcodec",
"libx264",
"-preset",
"slow",
"-crf",
"18",
]
if overwrite:
args.append("-y")
args.append(output_path)
try:
subprocess.check_call(args, cwd=tmp_dir)
except subprocess.CalledProcessError as e:
print(
"""
Could not call `ffmpeg` to convert your frames into a video.
If you want to convert the frames manually, you can extract the
.tar archive into a directory, cd to that directory, and run:
ffmpeg -r 60 -i %07d.png \\\n\t -vcodec libx264 \\\n\t -preset slow \\\n\t -crf 18 \\\n\t output.mp4
"""
)
raise
print("Saved output as {:s}".format(output_path))
return output_path
-49
View File
@@ -1,49 +0,0 @@
#!/usr/bin/env python
import os
import sys
import subprocess
import time
def file_filter(name):
return (not name.startswith(".")) and (not name.endswith(".swp"))
def file_times(path):
for top_level in filter(file_filter, os.listdir(path)):
for root, dirs, files in os.walk(top_level):
for file in filter(file_filter, files):
yield os.stat(os.path.join(root, file)).st_mtime
def print_stdout(process):
stdout = process.stdout
if stdout != None:
print(stdout)
# We concatenate all of the arguments together, and treat that as the command to run
command = " ".join(sys.argv[1:])
# The path to watch
path = "."
# How often we check the filesystem for changes (in seconds)
wait = 1
# The process to autoreload
process = subprocess.Popen(command, shell=True)
# The current maximum file modified time under the watched directory
last_mtime = max(file_times(path))
while True:
max_mtime = max(file_times(path))
print_stdout(process)
if max_mtime > last_mtime:
last_mtime = max_mtime
process.kill()
print("Restarting process.")
process = subprocess.Popen(command, shell=True)
time.sleep(wait)
-77
View File
@@ -19,83 +19,6 @@ from . import geometry as g
import numpy as np
import cv2
"""
TODO(ethan): add some notes about the coordinate conventions of the cameras
"""
class Camera(SceneElement):
def __init__(self):
super().__init__()
class OrthographicCamera(Camera):
def __init__(self, left, right, top, bottom, near, far, zoom=1):
super().__init__()
self.left = left
self.right = right
self.top = top
self.bottom = bottom
self.near = near
self.far = far
self.zoom = zoom
def lower(self):
data = {
"object": {
"uuid": self.uuid,
"type": "OrthographicCamera",
"left": self.left,
"right": self.right,
"top": self.top,
"bottom": self.bottom,
"near": self.near,
"far": self.far,
"zoom": self.zoom,
}
}
return data
class PerspectiveCamera(Camera):
def __init__(self, aspect, fov, near, far, zoom=1):
super().__init__()
self.aspect = aspect
self.fov = fov
self.near = near
self.far = far
@staticmethod
def from_intrinsics(focal_length, aspect_ratio):
"""TODO(ethan): finish this"""
# convert the intrinsics matrix into three.js compatible format
return PerspectiveCamera(aspect, fov, near, far)
def lower(self):
data = {
"object": {
"uuid": self.uuid,
"type": "PerspectiveCamera",
"aspect": self.aspect,
"fov": self.fov,
"near": self.near,
"far": self.far,
}
}
return data
class CameraHelper(Camera):
"""NOTE(ethan): Camera parent class might not make sense here?"""
def __init__(self, camera):
super().__init__()
self.camera = camera
def lower(self):
data = {"object": {"uuid": self.uuid, "type": "CameraHelper", "camera": self.camera.lower()}}
return data
class ImagePlane(g.Mesh):
def __init__(self, image, height=1, width=1):
+30 -29
View File
@@ -29,7 +29,6 @@ from .geometry import (
PointsMaterial,
Points,
)
from .cameras import Camera, CameraHelper
from .path import Path
@@ -59,6 +58,36 @@ class SetObject:
return {"type": "set_object", "object": self.object.lower(), "path": self.path.lower()}
class GetObject:
__slots__ = ["path"]
def __init__(self, path):
self.path = path
def lower(self):
return {"type": "get_object", "path": self.path.lower()}
class SetImage:
"""
Sets the image for showing with web rtc.
"""
__slots__ = ["image", "path"]
def __init__(self, image, path):
self.image = image
self.path = path
def lower(self):
return {
"type": "set_image",
"path": self.path.lower(),
"image": list(self.image.flatten()),
"shape": list(self.image.shape),
}
class SetTransform:
__slots__ = ["matrix", "path"]
@@ -90,31 +119,3 @@ class SetProperty:
def lower(self):
return {"type": "set_property", "path": self.path.lower(), "property": self.key.lower(), "value": self.value}
class SetAnimation:
__slots__ = ["animation", "play", "repetitions"]
def __init__(self, animation, play=True, repetitions=1):
self.animation = animation
self.play = play
self.repetitions = repetitions
def lower(self):
return {
"type": "set_animation",
"animations": self.animation.lower(),
"options": {"play": self.play, "repetitions": self.repetitions},
"path": "",
}
class SetImage:
__slots__ = ["image", "path"]
def __init__(self, image, path):
self.image = image
self.path = path
def lower(self):
return {"type": "set_image", "image": self.image, "path": self.path.lower()}
+95 -40
View File
@@ -15,8 +15,10 @@
from __future__ import absolute_import, division, print_function
import base64
import json
import re
import sys
import numpy as np
if sys.version_info >= (3, 0):
ADDRESS_IN_USE_ERROR = OSError
@@ -29,11 +31,16 @@ import tornado.gen
import tornado.ioloop
import tornado.web
import tornado.websocket
import umsgpack
import zmq
import zmq.eventloop.ioloop
from pyrad.viewer.backend.tree import SceneTree, find_node, walk
from aiortc import RTCPeerConnection, RTCSessionDescription, VideoStreamTrack
from aiortc.rtcrtpsender import RTCRtpSender
from zmq.eventloop.zmqstream import ZMQStream
from pyrad.viewer.backend.tree import SceneTree, find_node, walk
from pyrad.viewer.backend.video_stream import FlagVideoStreamTrack, SingleFrameStreamTrack
def capture(pattern, s):
match = re.match(pattern, s)
@@ -66,7 +73,8 @@ MAX_ATTEMPTS = 1000
DEFAULT_ZMQ_METHOD = "tcp"
DEFAULT_ZMQ_PORT = 6000
DEFAULT_WEBSOCKET_PORT = 8051
MESHCAT_COMMANDS = ["set_transform", "set_object", "delete", "set_property", "set_animation", "set_image"]
WEBSOCKET_COMMANDS = ["set_transform", "set_object", "get_object", "set_property", "delete"]
WEBRTC_COMMANDS = ["set_image"]
def find_available_port(func, default_port, max_attempts=MAX_ATTEMPTS, **kwargs):
@@ -89,6 +97,13 @@ def find_available_port(func, default_port, max_attempts=MAX_ATTEMPTS, **kwargs)
)
def force_codec(pc, sender, forced_codec):
kind = forced_codec.split("/")[0]
codecs = RTCRtpSender.getCapabilities(kind).codecs
transceiver = next(t for t in pc.getTransceivers() if t.sender == sender)
transceiver.setCodecPreferences([codec for codec in codecs if codec.mimeType == forced_codec])
class WebSocketHandler(tornado.websocket.WebSocketHandler):
def __init__(self, *args, **kwargs):
self.bridge = kwargs.pop("bridge")
@@ -103,25 +118,51 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler):
print("opened:", self, file=sys.stderr)
self.bridge.send_scene(self)
def on_message(self, message):
print(message)
async def on_message(self, message):
data = message
m = umsgpack.unpackb(message)
type_ = m["type"]
path = list(filter(lambda x: len(x) > 0, m["path"].split("/")))
if type_ == "set_transform":
find_node(self.bridge.tree, path).transform = data
elif type_ == "set_object":
find_node(self.bridge.tree, path).object = data
find_node(self.bridge.tree, path).properties = []
elif type_ == "offer":
print("making an offer")
print("sending an answer")
# print(m)
offer = RTCSessionDescription(m["data"]["sdp"], m["data"]["type"])
pc = RTCPeerConnection()
self.bridge.pcs.add(pc) # TODO(ethan): handle this better, since this set will get large
# video = FlagVideoStreamTrack()
video = SingleFrameStreamTrack()
self.bridge.video_tracks.add(video)
video_sender = pc.addTrack(video)
# force_codec(this.bridge.pc, video_sender, video_codec)
await pc.setRemoteDescription(offer)
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
cmd_data = {
"type": "answer",
"path": "",
"data": {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type},
}
data = umsgpack.packb(cmd_data)
self.write_message(data, binary=True)
def on_close(self):
self.bridge.websocket_pool.remove(self)
print("closed:", self, file=sys.stderr)
def create_command(data):
"""Encode the drawing command into a Javascript fetch() command for display."""
return """
fetch("data:application/octet-binary;base64,{}")
.then(res => res.arrayBuffer())
.then(buffer => viewer.handle_command_bytearray(new Uint8Array(buffer)));
""".format(
base64.b64encode(data).decode("utf-8")
)
class ZMQWebSocketBridge(object):
context = zmq.Context()
@@ -130,6 +171,8 @@ class ZMQWebSocketBridge(object):
self.websocket_pool = set()
self.app = self.make_app()
self.ioloop = tornado.ioloop.IOLoop.current()
self.pcs = set()
self.video_tracks = set()
if zmq_url is None:
@@ -155,44 +198,58 @@ class ZMQWebSocketBridge(object):
return f"{class_name} using zmq_url={self.zmq_url} and websocket_port={self.websocket_port}"
def make_app(self):
"""Create a tornado application for the websocket server."""
return tornado.web.Application([(r"/", WebSocketHandler, {"bridge": self})])
def handle_zmq(self, frames):
cmd = frames[0].decode("utf-8")
if cmd not in MESHCAT_COMMANDS:
print(f"{cmd} not in MESHCAT_COMMANDS! Proceeding anyways.")
# self.zmq_socket.send(b"error: unrecognized comand")
print(cmd)
if len(frames) != 3:
self.zmq_socket.send(b"error: expected 3 frames")
return
path = list(filter(lambda x: len(x) > 0, frames[1].decode("utf-8").split("/")))
data = frames[2]
self.forward_to_websockets(frames)
if cmd == "set_transform":
find_node(self.tree, path).transform = data
elif cmd == "set_object":
find_node(self.tree, path).object = data
find_node(self.tree, path).properties = []
elif cmd == "set_property":
find_node(self.tree, path).properties.append(data)
elif cmd == "set_animation":
find_node(self.tree, path).animation = data
elif cmd == "delete":
if len(path) > 0:
parent = find_node(self.tree, path[:-1])
child = path[-1]
if child in parent:
del parent[child]
else:
self.tree = SceneTree()
if cmd in WEBSOCKET_COMMANDS:
self.forward_to_websockets(frames)
if cmd == "set_transform":
find_node(self.tree, path).transform = data
elif cmd == "set_object":
find_node(self.tree, path).object = data
find_node(self.tree, path).properties = []
elif cmd == "get_object":
data = find_node(self.tree, path).object
self.zmq_socket.send(data)
return
elif cmd == "set_property":
find_node(self.tree, path).properties.append(data)
elif cmd == "delete":
if len(path) > 0:
parent = find_node(self.tree, path[:-1])
child = path[-1]
if child in parent:
del parent[child]
else:
self.tree = SceneTree()
elif cmd in WEBRTC_COMMANDS:
if cmd == "set_image":
for video_track in self.video_tracks:
unpacked_data = umsgpack.unpackb(data)
image = np.array(unpacked_data["image"], dtype="uint8").reshape(unpacked_data["shape"])
video_track.put_frame(image)
else:
self.zmq_socket.send(b"error: unknown command")
return
self.zmq_socket.send(b"ok")
return
def forward_to_websockets(self, frames):
cmd, path, data = frames
"""Forward a zmq message to all websockets."""
_, _, data = frames # cmd, path, data
for websocket in self.websocket_pool:
websocket.write_message(data, binary=True)
def setup_zmq(self, url):
"""Setup a zmq socket and connect it to the given url."""
zmq_socket = self.context.socket(zmq.REP)
zmq_socket.bind(url)
zmq_stream = ZMQStream(zmq_socket)
@@ -207,8 +264,6 @@ class ZMQWebSocketBridge(object):
websocket.write_message(p, binary=True)
if node.transform is not None:
websocket.write_message(node.transform, binary=True)
if node.animation is not None:
websocket.write_message(node.animation, binary=True)
def run(self):
self.ioloop.start()
@@ -217,7 +272,7 @@ class ZMQWebSocketBridge(object):
def main():
import argparse
parser = argparse.ArgumentParser(description="Serve the MeshCat HTML files and listen for ZeroMQ commands")
parser = argparse.ArgumentParser(description="Listen for ZeroMQ commands")
parser.add_argument("--zmq-url", "-z", type=str, nargs="?", default=None)
parser.add_argument("--websocket-port", "-wp", type=str, nargs="?", default=None)
args = parser.parse_args()
+51
View File
@@ -0,0 +1,51 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import numpy as np
import torch
def get_chunks(lst, num_chunks=None, size_of_chunk=None):
"""Returns list of n elements, constaining a sublist."""
if num_chunks:
assert not size_of_chunk
size = len(lst) // num_chunks
if size_of_chunk:
assert not num_chunks
size = size_of_chunk
chunks = []
for i in range(0, len(lst), size):
chunks.append(lst[i : i + size])
return chunks
def get_intrinsics_matrix_and_camera_to_world_h(camera_object, image_height):
"""Returns the camera intrinsics matrix and the camera to world homogeneous matrix.
Args:
camera_object: a Camera object.
image_size: the size of the image (height, width)
"""
# intrinsics
fov = camera_object["fov"]
aspect = camera_object["aspect"]
image_width = aspect * image_height
pp_w = image_width / 2.0
pp_h = image_height / 2.0
focal_length = pp_h / np.tan(fov * (np.pi / 180.0) / 2.0)
intrinsics_matrix = torch.tensor([[focal_length, 0, pp_w], [0, focal_length, pp_h], [0, 0, 1]]).float()
# extrinsics
camera_to_world_h = torch.tensor(get_chunks(camera_object["matrix"], size_of_chunk=4)).T.float()
return intrinsics_matrix, camera_to_world_h
+93
View File
@@ -0,0 +1,93 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import asyncio
import queue
import math
import cv2
import numpy
from aiortc import VideoStreamTrack
from av import VideoFrame
class FlagVideoStreamTrack(VideoStreamTrack):
"""
A video track that returns an animated flag.
"""
def __init__(self):
super().__init__() # don't forget this!
self.counter = 0
height, width = 480, 640
# generate flag
data_bgr = numpy.hstack(
[
self._create_rectangle(width=213, height=480, color=(255, 0, 0)), # blue
self._create_rectangle(width=214, height=480, color=(255, 255, 255)), # white
self._create_rectangle(width=213, height=480, color=(0, 0, 255)), # red
]
)
# shrink and center it
M = numpy.float32([[0.5, 0, width / 4], [0, 0.5, height / 4]])
data_bgr = cv2.warpAffine(data_bgr, M, (width, height))
# compute animation
omega = 2 * math.pi / height
id_x = numpy.tile(numpy.array(range(width), dtype=numpy.float32), (height, 1))
id_y = numpy.tile(numpy.array(range(height), dtype=numpy.float32), (width, 1)).transpose()
self.frames = []
for k in range(30):
phase = 2 * k * math.pi / 30
map_x = id_x + 10 * numpy.cos(omega * id_x + phase)
map_y = id_y + 10 * numpy.sin(omega * id_x + phase)
frame = cv2.remap(data_bgr, map_x, map_y, cv2.INTER_LINEAR)
self.frames.append(VideoFrame.from_ndarray(frame, format="bgr24"))
async def recv(self):
pts, time_base = await self.next_timestamp()
frame = self.frames[self.counter % 30]
frame.pts = pts
frame.time_base = time_base
self.counter += 1
return frame
def _create_rectangle(self, width, height, color):
data_bgr = numpy.zeros((height, width, 3), numpy.uint8)
data_bgr[:, :] = color
return data_bgr
class SingleFrameStreamTrack(VideoStreamTrack):
def __init__(self):
super().__init__()
self.background_frame = numpy.zeros((480, 640, 3), dtype="uint8") * 255
self.frame = None
self.put_frame(self.background_frame)
def put_frame(self, frame):
self.frame = VideoFrame.from_ndarray(frame)
async def recv(self):
pts, time_base = await self.next_timestamp()
frame = self.frame
frame.pts = pts
frame.time_base = time_base
return frame
+213
View File
@@ -0,0 +1,213 @@
# Copyright 2022 The Plenoptix Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Client class"""
import argparse
import asyncio
import json
import logging
import math
import os
import ssl
import aiohttp_cors
import cv2
import numpy
from aiohttp import web
from aiortc import RTCPeerConnection, RTCSessionDescription, VideoStreamTrack
from aiortc.rtcrtpsender import RTCRtpSender
from av import VideoFrame
ROOT = os.path.dirname(__file__)
class SingleFrameStreamTrack(VideoStreamTrack):
def __init__(self):
super().__init__() # don't forget this!
self.last_frame = None
self.frame_queue = asyncio.Queue(10)
async def put_frame(self, frame):
await self.frame_queue.put(VideoFrame.from_ndarray(frame))
async def recv(self):
if self.last_frame is None:
frame = await self.frame_queue.get()
else:
try:
frame = self.frame_queue.get_nowait()
except asyncio.QueueEmpty:
frame = self.last_frame
pts, time_base = await self.next_timestamp()
frame.pts = pts
frame.time_base = time_base
self.last_frame = frame
return frame
def _create_rectangle(width, height, color):
data_bgr = numpy.zeros((height, width, 3), numpy.uint8)
data_bgr[:, :] = color
return data_bgr
def _gen_test_image(k):
height, width = 480, 640
# generate flag
data_bgr = numpy.hstack(
[
_create_rectangle(width=213, height=480, color=(255, 0, 0)), # blue
_create_rectangle(width=214, height=480, color=(255, 255, 255)), # white
_create_rectangle(width=213, height=480, color=(0, 0, 255)), # red
]
)
# shrink and center it
M = numpy.float32([[0.5, 0, width / 4], [0, 0.5, height / 4]])
data_bgr = cv2.warpAffine(data_bgr, M, (width, height))
# compute animation
omega = 2 * math.pi / height
id_x = numpy.tile(numpy.array(range(width), dtype=numpy.float32), (height, 1))
id_y = numpy.tile(numpy.array(range(height), dtype=numpy.float32), (width, 1)).transpose()
phase = 2 * k * math.pi / 30
map_x = id_x + 10 * numpy.cos(omega * id_x + phase)
map_y = id_y + 10 * numpy.sin(omega * id_x + phase)
return cv2.remap(data_bgr, map_x, map_y, cv2.INTER_LINEAR)
async def gen_image_frames(video_track):
k = 0
while True:
if k > 500:
k = 0
img = _gen_test_image(k)
k += 1
await video_track.put_frame(img)
def force_codec(pc, sender, forced_codec):
kind = forced_codec.split("/")[0]
codecs = RTCRtpSender.getCapabilities(kind).codecs
transceiver = next(t for t in pc.getTransceivers() if t.sender == sender)
transceiver.setCodecPreferences([codec for codec in codecs if codec.mimeType == forced_codec])
async def offer(request):
params = await request.json()
offer = RTCSessionDescription(sdp=params["sdp"], type=params["type"])
pc = RTCPeerConnection()
pcs.add(pc)
@pc.on("connectionstatechange")
async def on_connectionstatechange():
print("Connection state is %s" % pc.connectionState)
if pc.connectionState == "failed":
await pc.close()
pcs.discard(pc)
video = SingleFrameStreamTrack()
loop = asyncio.get_event_loop()
try:
asyncio.run_coroutine_threadsafe(gen_image_frames(video), loop)
except KeyboardInterrupt:
pass
finally:
pc.close()
if video:
video_sender = pc.addTrack(video)
if args.video_codec:
force_codec(pc, video_sender, args.video_codec)
elif args.play_without_decoding:
raise Exception("You must specify the video codec using --video-codec")
await pc.setRemoteDescription(offer)
answer = await pc.createAnswer()
await pc.setLocalDescription(answer)
return web.Response(
content_type="application/json",
text=json.dumps({"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}),
)
pcs = set()
async def on_shutdown(app):
# close peer connections
coros = [pc.close() for pc in pcs]
await asyncio.gather(*coros)
pcs.clear()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="WebRTC webcam demo")
parser.add_argument("--cert-file", help="SSL certificate file (for HTTPS)")
parser.add_argument("--key-file", help="SSL key file (for HTTPS)")
parser.add_argument(
"--play-without-decoding",
help=(
"Read the media without decoding it (experimental). "
"For now it only works with an MPEGTS container with only H.264 video."
),
action="store_true",
)
parser.add_argument("--host", default="0.0.0.0", help="Host for HTTP server (default: 0.0.0.0)")
parser.add_argument("--port", type=int, default=8052, help="Port for HTTP server (default: 8080)")
parser.add_argument("--verbose", "-v", action="count")
parser.add_argument("--audio-codec", help="Force a specific audio codec (e.g. audio/opus)")
parser.add_argument("--video-codec", help="Force a specific video codec (e.g. video/H264)")
args = parser.parse_args()
if args.verbose:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
if args.cert_file:
ssl_context = ssl.SSLContext()
ssl_context.load_cert_chain(args.cert_file, args.key_file)
else:
ssl_context = None # pylint: disable=invalid-name
app = web.Application()
app.router.add_route("POST", "/offer", offer)
app.on_shutdown.append(on_shutdown)
cors = aiohttp_cors.setup(
app,
defaults={
"*": aiohttp_cors.ResourceOptions(
allow_credentials=True,
expose_headers="*",
allow_headers="*",
)
},
)
for route in list(app.router.routes()):
cors.add(route)
web.run_app(app, host=args.host, port=args.port, ssl_context=ssl_context)
+8
View File
@@ -217,3 +217,11 @@ def set_camera(vis, camera: Camera):
pose = camera.get_camera_to_world_h()
K = camera.get_intrinsics_matrix()
set_persp_camera(vis, pose=pose.double().numpy(), K=K.double().numpy())
def draw_aabb(vis, aabb, name="aabb"):
"""Draw the axis-aligned bounding box."""
lengths = aabb[1] - aabb[0]
vis[name].set_object(g.Box(lengths.tolist()), material=g.MeshPhongMaterial(color=0xFF0000, opacity=0.1))
center = aabb[0] + lengths / 2.0
vis[name].set_transform(tf.translation_matrix(center.tolist()))
+9 -14
View File
@@ -14,14 +14,11 @@
from __future__ import absolute_import, division, print_function
import sys
import time
import numpy as np
import umsgpack
import zmq
from .commands import Delete, SetAnimation, SetImage, SetObject, SetProperty, SetTransform
from .commands import Delete, SetObject, GetObject, SetImage, SetProperty, SetTransform
from .path import Path
@@ -42,19 +39,17 @@ class ViewerWindow(object):
umsgpack.packb(cmd_data),
]
)
self.client.recv()
return self.client.recv()
class Visualizer(object):
__slots__ = ["window", "path"]
def __init__(self, window):
self.window = window
self.path = Path(("pyrad",)) # TODO(ethan): change this
@staticmethod
def view_into(window, path):
vis = Visualizer(window=window)
vis = Visualizer(window)
vis.path = path
return vis
@@ -64,6 +59,12 @@ class Visualizer(object):
def set_object(self, geometry, material=None):
return self.window.send(SetObject(geometry, material, self.path))
def get_object(self):
return self.window.send(GetObject(self.path))
def set_image(self, image):
return self.window.send(SetImage(image, self.path))
def set_transform(self, matrix=np.eye(4)):
assert matrix.shape == (4, 4)
return self.window.send(SetTransform(matrix, self.path))
@@ -71,12 +72,6 @@ class Visualizer(object):
def set_property(self, key, value):
return self.window.send(SetProperty(key, value, self.path))
def set_animation(self, animation, play=True, repetitions=1):
return self.window.send(SetAnimation(animation, play=play, repetitions=repetitions))
def set_image(self, image):
return self.window.send(SetImage(image, self.path))
def delete(self):
return self.window.send(Delete(self.path))