mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 22:56:03 +08:00
CropBox GUI implementation (#1205)
* fixed an exception bug in case of num_samples is 0 fixed an exception bug when num samples is 0 , "IndexError tensors used as indices must be long, byte or bool tensors" change type to long ( was float32) * implemented crop box in gui. For now works on instant-ngp-bounded * fixed import bugs * update aabb every time and not only on training * prevent crash when chosing accomulation and crop box * Checkbox * max_box is bounded by scevne box in the gui * implemented altenative intesection to nerfacc * convert str to bool correctly * fixed isort bug * fixed isort bug2 * update viewer version * remove json bug * Update UI * Force black background * Fix formatting Co-authored-by: dror <you@example.com> Co-authored-by: Matt Tancik <tancik@berkeley.edu> Co-authored-by: Matthew Tancik <mattt4001@gmail.com>
This commit is contained in:
12 files changed
+518
-25
No files matched your search
@@ -16,19 +16,25 @@
|
||||
Camera Models
|
||||
"""
|
||||
import base64
|
||||
import importlib
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from distutils.util import strtobool
|
||||
from enum import Enum, auto
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
import cv2
|
||||
import torch
|
||||
import torchvision
|
||||
from torch.nn import Parameter
|
||||
from torchtyping import TensorType
|
||||
|
||||
import nerfstudio.utils.math
|
||||
import nerfstudio.utils.poses as pose_utils
|
||||
from nerfstudio.cameras import camera_utils
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.data.scene_box import SceneBox
|
||||
from nerfstudio.utils.tensor_dataclass import TensorDataclass
|
||||
|
||||
|
||||
@@ -308,6 +314,7 @@ class Cameras(TensorDataclass):
|
||||
distortion_params_delta: Optional[TensorType["num_rays":..., 6]] = None,
|
||||
keep_shape: Optional[bool] = None,
|
||||
disable_distortion: bool = False,
|
||||
aabb_box: SceneBox = None,
|
||||
) -> RayBundle:
|
||||
"""Generates rays for the given camera indices.
|
||||
|
||||
@@ -350,6 +357,7 @@ class Cameras(TensorDataclass):
|
||||
keeping dimensions. If False, we flatten at the end. If True, then we keep the shape of the
|
||||
camera_indices and coords tensors (if we can).
|
||||
disable_distortion: If True, disables distortion.
|
||||
aabb_box: if not None will calculate nears and fars of the ray according to aabb box intesection
|
||||
|
||||
Returns:
|
||||
Rays for the given camera indices and coords.
|
||||
@@ -449,6 +457,31 @@ class Cameras(TensorDataclass):
|
||||
if keep_shape is False:
|
||||
raybundle = raybundle.flatten()
|
||||
|
||||
if aabb_box:
|
||||
with torch.no_grad():
|
||||
tensor_aabb = Parameter(aabb_box.aabb.flatten(), requires_grad=False)
|
||||
|
||||
rays_o = raybundle.origins.contiguous()
|
||||
rays_d = raybundle.directions.contiguous()
|
||||
|
||||
tensor_aabb = tensor_aabb.to(rays_o.device)
|
||||
shape = rays_o.shape
|
||||
|
||||
rays_o = rays_o.reshape((-1, 3))
|
||||
rays_d = rays_d.reshape((-1, 3))
|
||||
|
||||
if strtobool(os.environ.get("INTERSECT_WITH_NERFACC", "TRUE")):
|
||||
nerfacc = importlib.import_module("nerfacc")
|
||||
t_min, t_max = nerfacc.ray_aabb_intersect(rays_o, rays_d, tensor_aabb)
|
||||
else:
|
||||
t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb)
|
||||
|
||||
t_min = t_min.reshape([shape[0], shape[1], 1])
|
||||
t_max = t_max.reshape([shape[0], shape[1], 1])
|
||||
|
||||
raybundle.nears = t_min
|
||||
raybundle.fars = t_max
|
||||
|
||||
# TODO: We should have to squeeze the last dimension here if we started with zero batch dims, but never have to,
|
||||
# so there might be a rogue squeeze happening somewhere, and this may cause some unintended behaviour
|
||||
# that we haven't caught yet with tests
|
||||
|
||||
@@ -449,6 +449,15 @@ class VolumetricSampler(Sampler):
|
||||
|
||||
rays_o = ray_bundle.origins.contiguous()
|
||||
rays_d = ray_bundle.directions.contiguous()
|
||||
|
||||
if ray_bundle.nears is not None and ray_bundle.fars is not None:
|
||||
t_min = ray_bundle.nears.contiguous().reshape(-1)
|
||||
t_max = ray_bundle.fars.contiguous().reshape(-1)
|
||||
|
||||
else:
|
||||
t_min = None
|
||||
t_max = None
|
||||
|
||||
if ray_bundle.camera_indices is not None:
|
||||
camera_indices = ray_bundle.camera_indices.contiguous()
|
||||
else:
|
||||
@@ -457,6 +466,8 @@ class VolumetricSampler(Sampler):
|
||||
ray_indices, starts, ends = nerfacc.ray_marching(
|
||||
rays_o=rays_o,
|
||||
rays_d=rays_d,
|
||||
t_min=t_min,
|
||||
t_max=t_max,
|
||||
scene_aabb=self.scene_aabb,
|
||||
grid=self.occupancy_grid,
|
||||
# this is a workaround - using density causes crash and damage quality. should be fixed
|
||||
|
||||
@@ -26,8 +26,9 @@ Example:
|
||||
rgb = rgb_renderer(rgb=field_outputs[FieldHeadNames.RGB], weights=weights)
|
||||
|
||||
"""
|
||||
import contextlib
|
||||
import math
|
||||
from typing import Optional, Union
|
||||
from typing import Generator, Optional, Union
|
||||
|
||||
import nerfacc
|
||||
import torch
|
||||
@@ -38,6 +39,20 @@ from typing_extensions import Literal
|
||||
from nerfstudio.cameras.rays import RaySamples
|
||||
from nerfstudio.utils.math import components_from_spherical_harmonics
|
||||
|
||||
BACKGROUND_MODE_OVERRIDE: Optional[Literal["last_sample", "black", "random"]] = None
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def background_mode_override_context(mode: Literal["last_sample", "black"]) -> Generator[None, None, None]:
|
||||
"""Context manager for setting background mode."""
|
||||
global BACKGROUND_MODE_OVERRIDE # pylint: disable=global-statement
|
||||
old_background_mode = BACKGROUND_MODE_OVERRIDE
|
||||
try:
|
||||
BACKGROUND_MODE_OVERRIDE = mode
|
||||
yield
|
||||
finally:
|
||||
BACKGROUND_MODE_OVERRIDE = old_background_mode
|
||||
|
||||
|
||||
class RGBRenderer(nn.Module):
|
||||
"""Standard volumetic rendering.
|
||||
@@ -55,7 +70,7 @@ class RGBRenderer(nn.Module):
|
||||
cls,
|
||||
rgb: TensorType["bs":..., "num_samples", 3],
|
||||
weights: TensorType["bs":..., "num_samples", 1],
|
||||
background_color: Union[Literal["random", "last_sample"], TensorType[3]] = "random",
|
||||
background_color: Union[Literal["random", "black", "last_sample"], TensorType[3]] = "random",
|
||||
ray_indices: Optional[TensorType["num_samples"]] = None,
|
||||
num_rays: Optional[int] = None,
|
||||
) -> TensorType["bs":..., 3]:
|
||||
@@ -81,10 +96,14 @@ class RGBRenderer(nn.Module):
|
||||
comp_rgb = torch.sum(weights * rgb, dim=-2)
|
||||
accumulated_weight = torch.sum(weights, dim=-2)
|
||||
|
||||
if BACKGROUND_MODE_OVERRIDE is not None:
|
||||
background_color = BACKGROUND_MODE_OVERRIDE
|
||||
if background_color == "last_sample":
|
||||
background_color = rgb[..., -1, :]
|
||||
if background_color == "random":
|
||||
background_color = torch.rand_like(comp_rgb).to(rgb.device)
|
||||
if background_color == "black":
|
||||
background_color = torch.zeros_like(comp_rgb).to(rgb.device)
|
||||
|
||||
assert isinstance(background_color, torch.Tensor)
|
||||
comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight)
|
||||
|
||||
@@ -74,6 +74,7 @@ class Model(nn.Module):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.scene_box = scene_box
|
||||
self.render_aabb = None # the box that we want to render - should be a subset of scene_box
|
||||
self.num_train_data = num_train_data
|
||||
self.kwargs = kwargs
|
||||
self.collider = None
|
||||
|
||||
@@ -36,6 +36,7 @@ def apply_colormap(image: TensorType["bs":..., 1], cmap="viridis") -> TensorType
|
||||
|
||||
colormap = cm.get_cmap(cmap)
|
||||
colormap = torch.tensor(colormap.colors).to(image.device) # type: ignore
|
||||
image = torch.nan_to_num(image, 0)
|
||||
image_long = (image * 255).long()
|
||||
image_long_min = torch.min(image_long)
|
||||
image_long_max = torch.max(image_long)
|
||||
|
||||
@@ -191,3 +191,49 @@ def expected_sin(x_means: torch.Tensor, x_vars: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
|
||||
return torch.exp(-0.5 * x_vars) * torch.sin(x_means)
|
||||
|
||||
|
||||
def intersect_aabb(origins, directions, aabb):
|
||||
"""
|
||||
pytorch implementation of ray intersection with AABB box
|
||||
:param origin: N,3 tensor of 3d positions
|
||||
:param direction: N,3 tensor of normalized directions
|
||||
:param aabb: 6,1 array of aabb box in the form of [x_min, y_min, z_min, x_max, y_max, z_max]
|
||||
:return: t_min, t_max - two tensors of shapes N,1 representing the intersection signed distance from
|
||||
the origin. t_min is clipped to 0 in case it is negative, in case of no intersection t_min is 1e10.
|
||||
"""
|
||||
|
||||
max_bound = 1e10
|
||||
|
||||
tx_min = (aabb[0] - origins[:, 0]) / directions[:, 0]
|
||||
tx_max = (aabb[3] - origins[:, 0]) / directions[:, 0]
|
||||
t_min = torch.min(tx_min, tx_max)
|
||||
t_max = torch.max(tx_min, tx_max)
|
||||
|
||||
ty_min_temp = (aabb[1] - origins[:, 1]) / directions[:, 1]
|
||||
ty_max_temp = (aabb[4] - origins[:, 1]) / directions[:, 1]
|
||||
ty_min = torch.min(ty_min_temp, ty_max_temp)
|
||||
ty_max = torch.max(ty_min_temp, ty_max_temp)
|
||||
|
||||
cond = torch.logical_or((t_min > ty_max), (ty_min > t_max))
|
||||
t_min = torch.where(cond, max_bound, t_min)
|
||||
t_max = torch.where(cond, max_bound, t_max)
|
||||
|
||||
t_min = torch.max(t_min, ty_min)
|
||||
t_max = torch.min(t_max, ty_max)
|
||||
|
||||
tz_min_temp = (aabb[2] - origins[:, 2]) / directions[:, 2]
|
||||
tz_max_temp = (aabb[5] - origins[:, 2]) / directions[:, 2]
|
||||
tz_min = torch.min(tz_min_temp, tz_max_temp)
|
||||
tz_max = torch.max(tz_min_temp, tz_max_temp)
|
||||
|
||||
cond = torch.logical_or((t_min > tz_max), tz_min > t_max)
|
||||
t_min = torch.where(cond, max_bound, t_min)
|
||||
t_max = torch.where(cond, max_bound, t_max)
|
||||
|
||||
t_min = torch.max(t_min, tz_min)
|
||||
t_max = torch.min(t_max, tz_max)
|
||||
t_min = torch.where(t_min > 0, t_min, 0)
|
||||
t_max = torch.where(t_min == max_bound, max_bound, t_max)
|
||||
|
||||
return t_min, t_max
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "viewer",
|
||||
"homepage": ".",
|
||||
"version": "23-01-19-0",
|
||||
"version": "23-01-20-0",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@emotion/react": "^11.10.4",
|
||||
|
||||
@@ -27,10 +27,17 @@ export function RenderControls() {
|
||||
const target_train_util = useSelector(
|
||||
(state) => state.renderingState.targetTrainUtil,
|
||||
);
|
||||
const render_time = useSelector(
|
||||
(state) => state.renderingState.renderTime,
|
||||
const render_time = useSelector((state) => state.renderingState.renderTime);
|
||||
const crop_enabled = useSelector(
|
||||
(state) => state.renderingState.crop_enabled,
|
||||
);
|
||||
|
||||
const box_size = useSelector((state) => state.renderingState.box_size);
|
||||
|
||||
const crop_scale = useSelector((state) => state.renderingState.crop_scale);
|
||||
|
||||
const crop_center = useSelector((state) => state.renderingState.crop_center);
|
||||
|
||||
const dispatch = useDispatch();
|
||||
|
||||
const set_max_resolution = (value) => {
|
||||
@@ -111,13 +118,13 @@ export function RenderControls() {
|
||||
|
||||
const [display_render_time, set_display_render_time] = useState(false);
|
||||
|
||||
const receive_temporal_dist = e => {
|
||||
const msg = msgpack.decode(new Uint8Array(e.data));
|
||||
if (msg.path === "/model/has_temporal_distortion") {
|
||||
set_display_render_time(msg.data === "true");
|
||||
websocket.removeEventListener("message", receive_temporal_dist);
|
||||
}
|
||||
}
|
||||
const receive_temporal_dist = (e) => {
|
||||
const msg = msgpack.decode(new Uint8Array(e.data));
|
||||
if (msg.path === '/model/has_temporal_distortion') {
|
||||
set_display_render_time(msg.data === 'true');
|
||||
websocket.removeEventListener('message', receive_temporal_dist);
|
||||
}
|
||||
};
|
||||
websocket.addEventListener('message', receive_temporal_dist);
|
||||
|
||||
const set_render_time = (value) => {
|
||||
@@ -139,6 +146,63 @@ export function RenderControls() {
|
||||
}
|
||||
};
|
||||
|
||||
const set_crop_enabled = (value) => {
|
||||
if (websocket.readyState === WebSocket.OPEN) {
|
||||
dispatch({
|
||||
type: 'write',
|
||||
path: 'renderingState/crop_enabled',
|
||||
data: value,
|
||||
});
|
||||
const cmd = 'write';
|
||||
const path = 'renderingState/crop_enabled';
|
||||
const data = {
|
||||
type: cmd,
|
||||
path,
|
||||
data: value,
|
||||
};
|
||||
const message = msgpack.encode(data);
|
||||
websocket.send(message);
|
||||
}
|
||||
};
|
||||
|
||||
const set_crop_scale = (value) => {
|
||||
if (websocket.readyState === WebSocket.OPEN) {
|
||||
dispatch({
|
||||
type: 'write',
|
||||
path: 'renderingState/crop_scale',
|
||||
data: value,
|
||||
});
|
||||
const cmd = 'write';
|
||||
const path = 'renderingState/crop_scale';
|
||||
const data = {
|
||||
type: cmd,
|
||||
path,
|
||||
data: value,
|
||||
};
|
||||
const message = msgpack.encode(data);
|
||||
websocket.send(message);
|
||||
}
|
||||
};
|
||||
|
||||
const set_crop_center = (value) => {
|
||||
if (websocket.readyState === WebSocket.OPEN) {
|
||||
dispatch({
|
||||
type: 'write',
|
||||
path: 'renderingState/crop_center',
|
||||
data: value,
|
||||
});
|
||||
const cmd = 'write';
|
||||
const path = 'renderingState/crop_center';
|
||||
const data = {
|
||||
type: cmd,
|
||||
path,
|
||||
data: value,
|
||||
};
|
||||
const message = msgpack.encode(data);
|
||||
websocket.send(message);
|
||||
}
|
||||
};
|
||||
|
||||
const [, setControls] = useControls(
|
||||
() => ({
|
||||
// training speed
|
||||
@@ -197,17 +261,51 @@ export function RenderControls() {
|
||||
'1024px': () => setControls({ max_resolution: 1024 }),
|
||||
'2048px': () => setControls({ max_resolution: 2048 }),
|
||||
}),
|
||||
// Dynamic NeRF rendering time
|
||||
...(display_render_time ? {render_time: {
|
||||
label: 'Render Timestep',
|
||||
value: render_time,
|
||||
min: 0,
|
||||
max: 1,
|
||||
step: 0.01,
|
||||
onChange: (v) => {
|
||||
set_render_time(v);
|
||||
// Enable Crop
|
||||
crop_options: {
|
||||
label: 'Crop Viewport',
|
||||
value: crop_enabled,
|
||||
onChange: (value) => {
|
||||
set_crop_enabled(value);
|
||||
},
|
||||
}} : {}),
|
||||
},
|
||||
crop_scale: {
|
||||
label: '| Scale',
|
||||
value: crop_scale,
|
||||
min: 0,
|
||||
max: 10,
|
||||
step: 0.05,
|
||||
render: (get) => get('crop_options'),
|
||||
onChange: (v) => {
|
||||
set_crop_scale(v);
|
||||
},
|
||||
},
|
||||
crop_center: {
|
||||
label: '| Center',
|
||||
value: crop_center,
|
||||
min: -10,
|
||||
max: 10,
|
||||
step: 0.05,
|
||||
render: (get) => get('crop_options'),
|
||||
onChange: (v) => {
|
||||
set_crop_center(v);
|
||||
},
|
||||
},
|
||||
// Dynamic NeRF rendering time
|
||||
...(display_render_time
|
||||
? {
|
||||
render_time: {
|
||||
label: 'Render Timestep',
|
||||
value: render_time,
|
||||
min: 0,
|
||||
max: 1,
|
||||
step: 0.01,
|
||||
onChange: (v) => {
|
||||
set_render_time(v);
|
||||
},
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
}),
|
||||
[
|
||||
outputOptions,
|
||||
@@ -215,6 +313,8 @@ export function RenderControls() {
|
||||
colormapOptions,
|
||||
colormapChoice,
|
||||
max_resolution,
|
||||
crop_enabled,
|
||||
box_size,
|
||||
target_train_util,
|
||||
render_time,
|
||||
display_render_time,
|
||||
|
||||
@@ -18,9 +18,13 @@ const msgpack = require('msgpack-lite');
|
||||
|
||||
function CameraToggle() {
|
||||
const dispatch = useDispatch();
|
||||
const websocket = useContext(WebSocketContext).socket;
|
||||
const camera_choice = useSelector(
|
||||
(state) => state.renderingState.camera_choice,
|
||||
);
|
||||
const crop_enabled = useSelector(
|
||||
(state) => state.renderingState.crop_enabled,
|
||||
);
|
||||
const set_camera_choice = (event: SelectChangeEvent, value: string[]) => {
|
||||
if (value != null) {
|
||||
dispatch({
|
||||
@@ -28,6 +32,21 @@ function CameraToggle() {
|
||||
path: 'renderingState/camera_choice',
|
||||
data: value,
|
||||
});
|
||||
if (websocket.readyState === WebSocket.OPEN) {
|
||||
const path = 'renderingState/crop_enabled';
|
||||
const data = {
|
||||
type: 'write',
|
||||
path,
|
||||
data: false,
|
||||
};
|
||||
if (value === 'Render Camera') {
|
||||
data.data = false;
|
||||
} else {
|
||||
data.data = crop_enabled;
|
||||
}
|
||||
const message = msgpack.encode(data);
|
||||
websocket.send(message);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -44,6 +44,11 @@ const initialState = {
|
||||
clipping_enabled: true,
|
||||
clipping_center: [0.0, 0.0, 0.0],
|
||||
clipping_box_scale: [2.0, 2.0, 2.0],
|
||||
|
||||
// Crop Box Options
|
||||
crop_enabled: false,
|
||||
crop_scale: [2.0, 2.0, 2.0],
|
||||
crop_center: [0.0, 0.0, 0.0],
|
||||
},
|
||||
// the scene state
|
||||
sceneState: {
|
||||
|
||||
@@ -42,6 +42,8 @@ from nerfstudio.cameras.cameras import Cameras, CameraType
|
||||
from nerfstudio.cameras.rays import RayBundle
|
||||
from nerfstudio.configs import base_config as cfg
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.data.scene_box import SceneBox
|
||||
from nerfstudio.model_components import renderers
|
||||
from nerfstudio.models.base_model import Model
|
||||
from nerfstudio.utils import colormaps, profiler, writer
|
||||
from nerfstudio.utils.decorators import check_main_thread, decorate_all
|
||||
@@ -144,8 +146,12 @@ class RenderThread(threading.Thread):
|
||||
outputs = None
|
||||
try:
|
||||
with SetTrace(self.state.check_interrupt):
|
||||
with torch.no_grad():
|
||||
outputs = self.graph.get_outputs_for_camera_ray_bundle(self.camera_ray_bundle)
|
||||
if self.state.prev_crop_enabled:
|
||||
with renderers.background_mode_override_context("black"), torch.no_grad():
|
||||
outputs = self.graph.get_outputs_for_camera_ray_bundle(self.camera_ray_bundle)
|
||||
else:
|
||||
with torch.no_grad():
|
||||
outputs = self.graph.get_outputs_for_camera_ray_bundle(self.camera_ray_bundle)
|
||||
except Exception as e: # pylint: disable=broad-except
|
||||
self.exc = e
|
||||
|
||||
@@ -217,6 +223,13 @@ class CheckThread(threading.Thread):
|
||||
self.state.check_interrupt_vis = True
|
||||
return
|
||||
|
||||
# check crop changes
|
||||
crop_enabled = self.state.vis["renderingState/crop_enabled"].read()
|
||||
if crop_enabled is not None:
|
||||
if self.state.prev_crop_enabled != crop_enabled:
|
||||
self.state.check_interrupt_vis = True
|
||||
return
|
||||
|
||||
|
||||
@decorate_all([check_main_thread])
|
||||
class ViewerState:
|
||||
@@ -272,6 +285,9 @@ class ViewerState:
|
||||
self.moving_fps = 24
|
||||
self.camera_moving = False
|
||||
self.prev_camera_timestamp = 0
|
||||
self.prev_crop_enabled = False
|
||||
self.prev_crop_scale = None
|
||||
self.prev_crop_center = None
|
||||
|
||||
self.output_list = None
|
||||
|
||||
@@ -335,6 +351,9 @@ class ViewerState:
|
||||
# set the initial state whether to train or not
|
||||
self.vis["renderingState/isTraining"].write(start_train)
|
||||
|
||||
max_scene_box = torch.max(dataset.scene_box.aabb[1] - dataset.scene_box.aabb[0]).item()
|
||||
self.vis["renderingState/max_box_size"].write(max_scene_box)
|
||||
|
||||
# self.vis["renderingState/render_time"].write(str(0))
|
||||
|
||||
# set the properties of the camera
|
||||
@@ -381,6 +400,48 @@ class ViewerState:
|
||||
# remove the offer from the state tree
|
||||
self.vis["webrtc/offer"].delete()
|
||||
|
||||
def _update_render_aabb(self, graph):
|
||||
"""
|
||||
update the render aabb box for the viewer:
|
||||
|
||||
:param graph:
|
||||
:return:
|
||||
"""
|
||||
|
||||
crop_enabled = self.vis["renderingState/crop_enabled"].read()
|
||||
if crop_enabled != self.prev_crop_enabled:
|
||||
self.camera_moving = True
|
||||
self.prev_crop_enabled = crop_enabled
|
||||
self.prev_crop_scale = None
|
||||
self.prev_crop_center = None
|
||||
|
||||
if crop_enabled:
|
||||
crop_scale = self.vis["renderingState/crop_scale"].read()
|
||||
crop_center = self.vis["renderingState/crop_center"].read()
|
||||
|
||||
if crop_scale != self.prev_crop_scale or crop_center != self.prev_crop_center:
|
||||
self.camera_moving = True
|
||||
self.prev_crop_scale = crop_scale
|
||||
self.prev_crop_center = crop_center
|
||||
|
||||
crop_scale = torch.tensor(crop_scale)
|
||||
crop_center = torch.tensor(crop_center)
|
||||
|
||||
box_min = crop_center - crop_scale / 2.0
|
||||
box_max = crop_center + crop_scale / 2.0
|
||||
|
||||
if isinstance(graph.render_aabb, SceneBox):
|
||||
graph.render_aabb.aabb[0] = box_min
|
||||
graph.render_aabb.aabb[1] = box_max
|
||||
else:
|
||||
graph.render_aabb = SceneBox(aabb=torch.stack([box_min, box_max], dim=0))
|
||||
|
||||
# maybe should update only if true change ?
|
||||
json_ = graph.render_aabb.to_json()
|
||||
self.vis["sceneState/sceneBox"].write(json_)
|
||||
else:
|
||||
graph.render_aabb = None
|
||||
|
||||
def update_scene(self, trainer, step: int, graph: Model, num_rays_per_batch: int) -> None:
|
||||
"""updates the scene based on the graph weights
|
||||
|
||||
@@ -481,6 +542,16 @@ class ViewerState:
|
||||
if self.prev_colormap_type != colormap_type:
|
||||
self.camera_moving = True
|
||||
|
||||
crop_scale = self.vis["renderingState/crop_scale"].read()
|
||||
if self.prev_crop_enabled:
|
||||
if self.prev_crop_scale != crop_scale:
|
||||
self.camera_moving = True
|
||||
|
||||
crop_center = self.vis["renderingState/crop_center"].read()
|
||||
if self.prev_crop_enabled:
|
||||
if self.prev_crop_center != crop_center:
|
||||
self.camera_moving = True
|
||||
|
||||
return camera_object
|
||||
|
||||
def _apply_colormap(self, outputs: Dict[str, Any], colors: torch.Tensor = None, eps=1e-6):
|
||||
@@ -760,6 +831,15 @@ class ViewerState:
|
||||
colormap_type = ColormapTypes.INIT if colormap_type is None else colormap_type
|
||||
self.prev_colormap_type = colormap_type
|
||||
|
||||
# update render aabb
|
||||
try:
|
||||
self._update_render_aabb(graph)
|
||||
except RuntimeError as e:
|
||||
self.vis["renderingState/log_errors"].write("Got an Error while trying to update aabb crop")
|
||||
print(f"Error: {e}")
|
||||
|
||||
time.sleep(0.5) # sleep to allow buffer to reset
|
||||
|
||||
# Calculate camera pose and intrinsics
|
||||
try:
|
||||
image_height, image_width = self._calculate_image_res(camera_object, is_training)
|
||||
@@ -811,7 +891,7 @@ class ViewerState:
|
||||
)
|
||||
camera = camera.to(graph.device)
|
||||
|
||||
camera_ray_bundle = camera.generate_rays(camera_indices=0)
|
||||
camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=graph.render_aabb)
|
||||
|
||||
graph.eval()
|
||||
|
||||
|
||||
@@ -0,0 +1,178 @@
|
||||
"""
|
||||
Test AABB intersection
|
||||
"""
|
||||
import importlib
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
from distutils.util import strtobool
|
||||
|
||||
import torch
|
||||
|
||||
from nerfstudio.utils.math import intersect_aabb
|
||||
|
||||
test_device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
|
||||
|
||||
def _swap_minmax(x, y):
|
||||
if x < y:
|
||||
return x, y
|
||||
|
||||
return y, x
|
||||
|
||||
|
||||
def _get_random_aabb_box(max_box, device):
|
||||
"""
|
||||
generate random aabb box
|
||||
:param max_box:
|
||||
:param device:
|
||||
:return:
|
||||
"""
|
||||
x_min = random.uniform(-max_box, max_box)
|
||||
x_max = random.uniform(-max_box, max_box)
|
||||
|
||||
x_min, x_max = _swap_minmax(x_min, x_max)
|
||||
|
||||
y_min = random.uniform(-max_box, max_box)
|
||||
y_max = random.uniform(-max_box, max_box)
|
||||
|
||||
y_min, y_max = _swap_minmax(y_min, y_max)
|
||||
|
||||
z_min = random.uniform(-max_box, max_box)
|
||||
z_max = random.uniform(-max_box, max_box)
|
||||
|
||||
z_min, z_max = _swap_minmax(z_min, z_max)
|
||||
|
||||
return torch.tensor([x_min, y_min, z_min, x_max, y_max, z_max], dtype=torch.float32, device=device)
|
||||
|
||||
|
||||
def _get_random_rays(num_rays, max_box, aabb, device):
|
||||
"""
|
||||
genrate random rays that distribute in the general direction of the box.
|
||||
The implementation is naive for now.
|
||||
:param num_rays:
|
||||
:param max_box:
|
||||
:param aabb:
|
||||
:param device:
|
||||
:return:
|
||||
"""
|
||||
origins = []
|
||||
directions = []
|
||||
aabb_center = torch.tensor(
|
||||
[0.5 * (aabb[0] + aabb[3]), 0.5 * (aabb[1] + aabb[4]), 0.5 * (aabb[2] + aabb[5])],
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
|
||||
for _ in range(num_rays):
|
||||
x = random.uniform(-max_box, max_box)
|
||||
y = random.uniform(-max_box, max_box)
|
||||
z = random.uniform(-max_box, max_box)
|
||||
|
||||
origin = torch.tensor([x, y, z], dtype=torch.float32, device=device)
|
||||
dirs_to_cent = aabb_center - origin
|
||||
dirs_to_cent = dirs_to_cent / torch.linalg.norm(dirs_to_cent)
|
||||
phi = math.asin(dirs_to_cent[2])
|
||||
theta = torch.atan2(dirs_to_cent[1], dirs_to_cent[0])
|
||||
|
||||
error_ang_phi = random.gauss(0, 0.05)
|
||||
error_ang_theta = random.gauss(0, 0.1)
|
||||
|
||||
phi = phi + error_ang_phi
|
||||
theta = theta + error_ang_theta
|
||||
|
||||
x_dir = math.cos(phi) * math.cos(theta)
|
||||
y_dir = math.cos(phi) * math.sin(theta)
|
||||
z_dir = math.sin(phi)
|
||||
|
||||
origins.append([x, y, z])
|
||||
directions.append([x_dir, y_dir, z_dir])
|
||||
|
||||
origins = torch.tensor(origins, dtype=torch.float32, device=device)
|
||||
directions = torch.tensor(directions, dtype=torch.float32, device=device)
|
||||
|
||||
return origins, directions
|
||||
|
||||
|
||||
def _is_point_on_aabb_boundary(point, aabb):
|
||||
# check
|
||||
error_threshold = 0.001
|
||||
for dim in range(3):
|
||||
if point[dim] < (aabb[dim] - error_threshold) or point[dim] > (aabb[dim + 3] + error_threshold):
|
||||
return False
|
||||
# check that the points is near the boundary
|
||||
for side in range(2):
|
||||
for dim in range(3):
|
||||
if abs(point[dim] - aabb[side * 3 + dim]) < error_threshold:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _assert_points_on_aabb_boundary(points, aabb):
|
||||
for point in points:
|
||||
assert _is_point_on_aabb_boundary(point, aabb)
|
||||
|
||||
|
||||
def test_rays_intersection_are_on_aabb_boundary():
|
||||
"""
|
||||
test that org + dir*t_min and org + dir*t_max are on aabb boundary
|
||||
:return:
|
||||
"""
|
||||
random.seed(496)
|
||||
max_box_size = 100
|
||||
aabb = _get_random_aabb_box(max_box_size, test_device)
|
||||
num_rays = 500
|
||||
max_test_size = 1000
|
||||
origins, directions = _get_random_rays(num_rays, max_test_size, aabb, test_device)
|
||||
|
||||
t_min, t_max = intersect_aabb(origins, directions, aabb)
|
||||
|
||||
mask = t_min < 1e10
|
||||
t_min = t_min[mask]
|
||||
t_max = t_max[mask]
|
||||
origins = origins[mask]
|
||||
directions = directions[mask]
|
||||
|
||||
# assert that we don't do something trivial
|
||||
assert len(t_min) > 10
|
||||
|
||||
# assert that tmax is larger than t_min
|
||||
assert torch.all(t_max >= t_min)
|
||||
|
||||
positions_min = origins + directions * t_min.view(-1, 1)
|
||||
positions_max = origins + directions * t_max.view(-1, 1)
|
||||
|
||||
_assert_points_on_aabb_boundary(positions_min, aabb)
|
||||
_assert_points_on_aabb_boundary(positions_max, aabb)
|
||||
|
||||
|
||||
def test_equall_nerfacc():
|
||||
"""
|
||||
test that the output of intersect_aabb is close to nerfacc.intersect_aabb
|
||||
:return:
|
||||
"""
|
||||
check_nerf_acc = strtobool(os.environ.get("INTERSECT_WITH_NERFACC", "FALSE"))
|
||||
if check_nerf_acc:
|
||||
nerfacc = importlib.import_module("nerfacc")
|
||||
random.seed(8128)
|
||||
|
||||
max_box_size = 100
|
||||
aabb = _get_random_aabb_box(max_box_size, test_device)
|
||||
num_rays = 500
|
||||
max_test_size = 1000
|
||||
|
||||
origins, directions = _get_random_rays(num_rays, max_test_size, aabb, test_device)
|
||||
|
||||
# time1 = time.time()
|
||||
t_min, t_max = intersect_aabb(origins, directions, aabb)
|
||||
# time2 = time.time()
|
||||
|
||||
# time3 = time.time()
|
||||
t_min_nerfacc, t_max_nerfacc = nerfacc.ray_aabb_intersect(origins, directions, aabb)
|
||||
# time4 = time.time()
|
||||
|
||||
# print("pytorch ", time2-time1)
|
||||
# print("nerfacc ", time4-time3)
|
||||
|
||||
assert torch.allclose(t_min, t_min_nerfacc, rtol=0.001)
|
||||
assert torch.allclose(t_max, t_max_nerfacc, rtol=0.001)
|
||||
Reference in new issue
Block a user