diff --git a/nerfstudio/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx b/nerfstudio/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx
deleted file mode 100644
index 80d69523d..000000000
--- a/nerfstudio/viewer/app/src/modules/WebRtcWindow/WebRtcWindow.jsx
+++ /dev/null
@@ -1,186 +0,0 @@
-import React, { createContext, useContext, useEffect, useRef } from 'react';
-
-import { useDispatch } from 'react-redux';
-import { WebSocketContext } from '../WebSocket/WebSocket';
-
-const WebRtcContext = createContext(null);
-const msgpack = require('msgpack-lite');
-
-export { WebRtcContext };
-
-export default function WebRtcWindow() {
- const websocket = useContext(WebSocketContext).socket;
- const pcRef = useRef(null);
- const localVideoRef = useRef(null);
-
- const dispatch = useDispatch();
-
- const getRTCPeerConnection = () => {
- const pc = new RTCPeerConnection({
- iceServers: [
- { urls: 'stun:stun.l.google.com:19302' },
- {
- urls: 'stun:openrelay.metered.ca:80',
- },
- {
- urls: 'turn:openrelay.metered.ca:80',
- username: 'openrelayproject',
- credential: 'openrelayproject',
- },
- {
- urls: 'turn:openrelay.metered.ca:443',
- username: 'openrelayproject',
- credential: 'openrelayproject',
- },
- {
- urls: 'turn:openrelay.metered.ca:443?transport=tcp',
- username: 'openrelayproject',
- credential: 'openrelayproject',
- },
- ],
- });
- // connect video
- pc.addEventListener('track', (evt) => {
- if (evt.track.kind === 'video') {
- [localVideoRef.current.srcObject] = evt.streams; // uses array destructuring
- }
- });
- pc.addTransceiver('video', { direction: 'recvonly' });
-
- // for updating the status of the peer connection
- pc.oniceconnectionstatechange = () => {
- // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/connectionState
- console.log(`[webrtc] connectionState: ${pc.connectionState}`);
- if (
- pc.connectionState === 'connecting' ||
- pc.connectionState === 'connected'
- ) {
- console.log('[webrtc] connected');
- dispatch({
- type: 'write',
- path: 'webrtcState/isConnected',
- data: true,
- });
- } else {
- dispatch({
- type: 'write',
- path: 'webrtcState/isConnected',
- data: false,
- });
- }
- };
-
- pc.onclose = () => {
- dispatch({
- type: 'write',
- path: 'webrtcState/isConnected',
- data: false,
- });
- };
-
- return pc;
- };
-
- const sendOffer = () => {
- pcRef.current
- .createOffer()
- .then((offer) => {
- console.log('[webrtc] created offer');
- console.log(offer);
- return pcRef.current.setLocalDescription(offer);
- })
- .then(() => {
- // wait for ICE gathering to complete
- console.log('[webrtc] set local description');
- return new Promise((resolve) => {
- if (pcRef.current.iceGatheringState === 'complete') {
- console.log('[webrtc] ICE gathering complete');
- resolve();
- } else {
- const checkState = () => {
- console.log(
- `[webrtc] iceGatheringState: ${pcRef.current.iceGatheringState}`,
- );
- if (pcRef.current.iceGatheringState === 'complete') {
- pcRef.current.removeEventListener(
- 'icegatheringstatechange',
- checkState,
- );
- resolve();
- }
- };
- console.log(
- '[webrtc] adding listener for `icegatheringstatechange`',
- );
- pcRef.current.addEventListener(
- 'icegatheringstatechange',
- checkState,
- );
- }
- });
- })
- .then(() => {
- // send the offer
- console.log('[webrtc] sending offer');
- const offer = pcRef.current.localDescription;
- const cmd = 'write';
- const path = 'webrtc/offer';
- const data = {
- type: cmd,
- path,
- data: {
- sdp: offer.sdp,
- type: offer.type,
- },
- };
- const message = msgpack.encode(data);
- websocket.send(message);
- });
- };
-
- useEffect(() => {
- websocket.addEventListener('message', (originalCmd) => {
- // set the remote description when the offer is received
- const cmd = msgpack.decode(new Uint8Array(originalCmd.data));
- if (cmd.path === '/webrtc/answer') {
- console.log('[webrtc] received answer');
- const answer = cmd.data;
- console.log(answer);
- if (answer !== null) {
- pcRef.current.setRemoteDescription(answer);
- }
- }
- });
-
- websocket.addEventListener('open', () => {
- // connect webrtc when the websocket is connected
- pcRef.current = getRTCPeerConnection();
- console.log('[webrtc] starting process');
- sendOffer();
- });
-
- // kill the webrtc connection on dismount
- return () => {
- if (pcRef.current !== null) {
- dispatch({
- type: 'write',
- path: 'webrtcState/isConnected',
- data: false,
- });
- pcRef.current.close();
- }
- };
- }, [websocket]); // dependency to call this whenever the websocket changes
-
- return (
-
-
-
- );
-}
diff --git a/nerfstudio/viewer/app/src/reducer.js b/nerfstudio/viewer/app/src/reducer.js
index 76a21fbb8..5aefe9c8a 100644
--- a/nerfstudio/viewer/app/src/reducer.js
+++ b/nerfstudio/viewer/app/src/reducer.js
@@ -6,13 +6,13 @@ const initialState = {
isConnected: false,
websocket_url: 'ws://localhost:7007',
},
- // the webrtc connection state
- webrtcState: {
- isConnected: false,
- },
+
// for sending actual commands to the client
camera_path_payload: null,
populate_paths_payload: false,
+
+ render_img: null, // The rendered images
+
// the rendering state
renderingState: {
// cameras
@@ -65,9 +65,6 @@ const initialState = {
sceneBox: null,
cameras: null,
},
- // ------------------------------------------------------------------------
- // which of the state keys to synchronize with the bridge server
- sync_keys: ['renderingState', 'sceneState'],
};
function setData(newState, state, path, data) {
diff --git a/nerfstudio/viewer/server/server.py b/nerfstudio/viewer/server/server.py
index 6c2c45476..dd2b62235 100644
--- a/nerfstudio/viewer/server/server.py
+++ b/nerfstudio/viewer/server/server.py
@@ -64,14 +64,10 @@ class WebSocketHandler(tornado.websocket.WebSocketHandler): # pylint: disable=a
if type_ == "write":
# writes the data coming from the websocket
find_node(self.bridge.state_tree, path).data = m["data"]
- if m["path"] != "renderingState/camera" and m["path"] != "webrtc/offer":
- # dispatch to the websockets if not a camera update!
- # TODO(ethan): handle the camera properly!
- # TODO: but don't update the current websocket
- command = {"type": "write", "path": m["path"], "data": m["data"]}
- packed_data = umsgpack.packb(command)
- frames = ["write".encode("utf-8"), m["path"].encode("utf-8"), packed_data]
- self.bridge.forward_to_websockets(frames, websocket_to_skip=self)
+ command = {"type": "write", "path": m["path"], "data": m["data"]}
+ packed_data = umsgpack.packb(command)
+ frames = ["write".encode("utf-8"), m["path"].encode("utf-8"), packed_data]
+ self.bridge.forward_to_websockets(frames, websocket_to_skip=self)
elif type_ == "read":
# reads and returns the data
data = find_node(self.bridge.state_tree, path).data
diff --git a/nerfstudio/viewer/server/utils.py b/nerfstudio/viewer/server/utils.py
index a21b02d82..b246790ef 100644
--- a/nerfstudio/viewer/server/utils.py
+++ b/nerfstudio/viewer/server/utils.py
@@ -21,8 +21,6 @@ from typing import Any, Callable, Dict, List, Optional, Tuple
import numpy as np
import torch
import zmq
-from aiortc import RTCPeerConnection
-from aiortc.rtcrtpsender import RTCRtpSender
def get_chunks(
@@ -127,17 +125,3 @@ def find_available_port(func: Callable, default_port: int, max_attempts: int = 1
raise (
Exception(f"Could not find an available port in the range: [{default_port:d}, {max_attempts + default_port:d})")
)
-
-
-def force_codec(pc: RTCPeerConnection, sender: RTCRtpSender, forced_codec: str) -> None:
- """Sets the codec preferences on a connection between sender and receiver
-
- Args:
- pc: peer connection point
- sender: sender that will send to connection point
- forced_codec: codec to set
- """
- kind = forced_codec.split("/")[0]
- codecs = RTCRtpSender.getCapabilities(kind).codecs
- transceiver = next(t for t in pc.getTransceivers() if t.sender == sender)
- transceiver.setCodecPreferences([codec for codec in codecs if codec.mimeType == forced_codec])
diff --git a/nerfstudio/viewer/server/video_stream.py b/nerfstudio/viewer/server/video_stream.py
deleted file mode 100644
index c50168055..000000000
--- a/nerfstudio/viewer/server/video_stream.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright 2022 The Nerfstudio Team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Video Stream objects for WebRTC"""
-
-import numpy as np
-from aiortc import VideoStreamTrack
-from av import VideoFrame
-
-
-class SingleFrameStreamTrack(VideoStreamTrack):
- """Single Frame stream class: pushes single frames to a stream"""
-
- def __init__(self):
- super().__init__()
- self.background_frame = np.ones((480, 640, 3), dtype="uint8") * 100 # gray background
- self.frame = None
- self.put_frame(self.background_frame)
-
- def put_frame(self, frame: np.ndarray) -> None:
- """Sets the current viewing frame
-
- Args:
- frame: image to be viewed
- """
- self.frame = VideoFrame.from_ndarray(frame)
-
- async def recv(self):
- """Async method to grab and wait on frame"""
- pts, time_base = await self.next_timestamp()
-
- frame = self.frame
- frame.pts = pts
- frame.time_base = time_base
- return frame
diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py
index 919a247e4..e13084f8e 100644
--- a/nerfstudio/viewer/server/viewer_utils.py
+++ b/nerfstudio/viewer/server/viewer_utils.py
@@ -18,7 +18,7 @@
"""
from __future__ import annotations
-import asyncio
+import base64
import enum
import os
import sys
@@ -28,14 +28,9 @@ import warnings
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
+import cv2
import numpy as np
import torch
-from aiortc import (
- RTCConfiguration,
- RTCIceServer,
- RTCPeerConnection,
- RTCSessionDescription,
-)
from cryptography.utils import CryptographyDeprecationWarning
from rich.console import Console
@@ -51,11 +46,7 @@ from nerfstudio.utils.decorators import check_main_thread, decorate_all
from nerfstudio.utils.io import load_from_json, write_to_json
from nerfstudio.utils.writer import GLOBAL_BUFFER, EventName, TimeWriter
from nerfstudio.viewer.server.subprocess import run_viewer_bridge_server_as_subprocess
-from nerfstudio.viewer.server.utils import (
- force_codec,
- get_intrinsics_matrix_and_camera_to_world_h,
-)
-from nerfstudio.viewer.server.video_stream import SingleFrameStreamTrack
+from nerfstudio.viewer.server.utils import get_intrinsics_matrix_and_camera_to_world_h
from nerfstudio.viewer.server.visualizer import Viewer
warnings.filterwarnings("ignore", category=CryptographyDeprecationWarning)
@@ -283,11 +274,9 @@ class ViewerState:
CONSOLE.rule(characters="=")
CONSOLE.line()
self.vis = Viewer(zmq_port=zmq_port, ip_address=self.config.ip_address)
- self.vis_webrtc_thread = Viewer(zmq_port=zmq_port, ip_address=self.config.ip_address)
else:
assert self.config.zmq_port is not None
self.vis = Viewer(zmq_port=self.config.zmq_port, ip_address=self.config.ip_address)
- self.vis_webrtc_thread = Viewer(zmq_port=self.config.zmq_port, ip_address=self.config.ip_address)
# viewer specific variables
self.prev_camera_matrix = None
@@ -314,12 +303,6 @@ class ViewerState:
self.output_list = None
- # webrtc
- self.pcs = set()
- self.video_tracks = set()
- self.webrtc_thread = None
- self.kill_webrtc_signal = False
-
def _pick_drawn_image_idxs(self, total_num: int) -> list[int]:
"""Determine indicies of images to display in viewer.
@@ -417,26 +400,6 @@ class ViewerState:
self.vis["renderingState/all_camera_paths"].write(all_path_dict)
self.vis["populate_paths_payload"].delete()
- def _check_webrtc_offer(self):
- """Check if there is a webrtc offer to respond to."""
- data = self.vis["webrtc/offer"].read()
- if data:
- if self.webrtc_thread and self.webrtc_thread.is_alive():
- # kill the previous thread if the webpage refreshes
- self.kill_webrtc_signal = True
- return
-
- def loop_in_thread(loop):
- asyncio.set_event_loop(loop)
- loop.run_until_complete(self.send_webrtc_answer(data))
-
- loop = asyncio.get_event_loop()
- self.webrtc_thread = threading.Thread(target=loop_in_thread, args=(loop,))
- self.webrtc_thread.daemon = True
- self.webrtc_thread.start()
- # remove the offer from the state tree
- self.vis["webrtc/offer"].delete()
-
def _update_render_aabb(self, graph):
"""
update the render aabb box for the viewer:
@@ -500,7 +463,6 @@ class ViewerState:
self._check_camera_path_payload(trainer, step)
self._check_populate_paths_payload(trainer, step)
- self._check_webrtc_offer()
camera_object = self._get_camera_object()
if camera_object is None:
@@ -546,7 +508,6 @@ class ViewerState:
is_training = self.vis["renderingState/isTraining"].read()
self._check_populate_paths_payload(trainer, step)
self._check_camera_path_payload(trainer, step)
- self._check_webrtc_offer()
run_loop = not is_training
local_step += 1
@@ -666,66 +627,6 @@ class ViewerState:
raise NotImplementedError
- async def send_webrtc_answer(self, data):
- """Setup the webrtc connection."""
-
- # returns the description to for WebRTC to the specific websocket connection
- offer = RTCSessionDescription(data["sdp"], data["type"])
-
- if self.config.local:
- pc = RTCPeerConnection()
- else:
- if self.config.skip_openrelay:
- ice_servers = [
- RTCIceServer(urls="stun:stun.l.google.com:19302"),
- ]
- else:
- ice_servers = [
- RTCIceServer(urls="stun:stun.l.google.com:19302"),
- RTCIceServer(urls="stun:openrelay.metered.ca:80"),
- RTCIceServer(
- urls="turn:openrelay.metered.ca:80", username="openrelayproject", credential="openrelayproject"
- ),
- RTCIceServer(
- urls="turn:openrelay.metered.ca:443", username="openrelayproject", credential="openrelayproject"
- ),
- RTCIceServer(
- urls="turn:openrelay.metered.ca:443?transport=tcp",
- username="openrelayproject",
- credential="openrelayproject",
- ),
- ]
-
- pc = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers))
- self.pcs.add(pc)
-
- video = SingleFrameStreamTrack()
- self.video_tracks.add(video)
- video_sender = pc.addTrack(video)
- print(f"viewer using video/{self.config.codec}")
- force_codec(pc, video_sender, f"video/{self.config.codec}")
-
- await pc.setRemoteDescription(offer)
- answer = await pc.createAnswer()
- await pc.setLocalDescription(answer)
-
- self.vis_webrtc_thread["webrtc/answer"].write(
- {"sdp": pc.localDescription.sdp, "type": pc.localDescription.type}
- )
- self.vis_webrtc_thread["webrtc/answer"].delete()
-
- # continually exchange media
- while True:
- await asyncio.sleep(1)
- if self.kill_webrtc_signal:
- self.kill_webrtc_signal = False
- return
-
- def set_image(self, image):
- """Write the image over webrtc."""
- for video_track in self.video_tracks:
- video_track.put_frame(image)
-
def _send_output_to_viewer(self, outputs: Dict[str, Any], colors: torch.Tensor = None):
"""Chooses the correct output and sends it to the viewer
@@ -760,8 +661,12 @@ class ViewerState:
self.vis["renderingState/colormap_choice"].write(self.prev_colormap_type)
self.vis["renderingState/colormap_options"].write(colormap_options)
selected_output = (self._apply_colormap(outputs, colors) * 255).type(torch.uint8)
- image = selected_output.cpu().numpy()
- self.set_image(image)
+
+ image = selected_output[..., [2, 1, 0]].cpu().numpy()
+
+ data = cv2.imencode(".jpg", image, [cv2.IMWRITE_JPEG_QUALITY, 75])[1].tobytes()
+ data = str("data:image/jpeg;base64," + base64.b64encode(data).decode("ascii"))
+ self.vis["render_img"].write(data)
def _update_viewer_stats(self, render_time: float, num_rays: int, image_height: int, image_width: int) -> None:
"""Function that calculates and populates all the rendering statistics accordingly
@@ -846,11 +751,6 @@ class ViewerState:
if image_width > self.max_resolution:
image_width = self.max_resolution
image_height = int(image_width / aspect_ratio)
- if self.config.codec != "VP8":
- # force even values to allow hardware encoder usage
- quantize = 2
- image_width = int(image_width / quantize) * quantize
- image_height = int(image_height / quantize) * quantize
return image_height, image_width
def _process_invalid_output(self, output_type: str) -> str:
@@ -880,7 +780,7 @@ class ViewerState:
# pylint: disable=too-many-statements
"""
Draw an image using the current camera pose from the viewer.
- The image is sent of a TCP connection and then uses WebRTC to send it to the viewer.
+ The image is sent over a TCP connection.
Args:
graph: current checkpoint of model
diff --git a/pyproject.toml b/pyproject.toml
index 314b22141..a89f966d8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,11 +14,9 @@ classifiers = [
"Programming Language :: Python",
]
dependencies = [
- "aiohttp>=3.8.1",
- "aiortc>=1.3.2",
"appdirs>=1.4",
"av>=9.2.0",
- "cryptography>=38,<39",
+ "cryptography>=38",
"tyro>=0.3.31",
"gdown>=4.6.0",
"ninja>=1.10",