From 2120bd5dd975857cbf1f70c20e20992b300639f8 Mon Sep 17 00:00:00 2001 From: dozeri83 <109915435+dozeri83@users.noreply.github.com> Date: Fri, 20 Jan 2023 03:39:37 +0200 Subject: [PATCH] CropBox GUI implementation (#1205) * fixed an exception bug in case of num_samples is 0 fixed an exception bug when num samples is 0 , "IndexError tensors used as indices must be long, byte or bool tensors" change type to long ( was float32) * implemented crop box in gui. For now works on instant-ngp-bounded * fixed import bugs * update aabb every time and not only on training * prevent crash when chosing accomulation and crop box * Checkbox * max_box is bounded by scevne box in the gui * implemented altenative intesection to nerfacc * convert str to bool correctly * fixed isort bug * fixed isort bug2 * update viewer version * remove json bug * Update UI * Force black background * Fix formatting Co-authored-by: dror Co-authored-by: Matt Tancik Co-authored-by: Matthew Tancik --- nerfstudio/cameras/cameras.py | 33 ++++ nerfstudio/model_components/ray_samplers.py | 11 ++ nerfstudio/model_components/renderers.py | 23 ++- nerfstudio/models/base_model.py | 1 + nerfstudio/utils/colormaps.py | 1 + nerfstudio/utils/math.py | 46 +++++ nerfstudio/viewer/app/package.json | 2 +- .../src/modules/ConfigPanel/ConfigPanel.jsx | 138 ++++++++++++-- .../src/modules/ViewerWindow/ViewerWindow.jsx | 19 ++ nerfstudio/viewer/app/src/reducer.js | 5 + nerfstudio/viewer/server/viewer_utils.py | 86 ++++++++- tests/utils/test_aabb_intersection.py | 178 ++++++++++++++++++ 12 files changed, 518 insertions(+), 25 deletions(-) create mode 100644 tests/utils/test_aabb_intersection.py diff --git a/nerfstudio/cameras/cameras.py b/nerfstudio/cameras/cameras.py index 68da40c82..f2020916f 100644 --- a/nerfstudio/cameras/cameras.py +++ b/nerfstudio/cameras/cameras.py @@ -16,19 +16,25 @@ Camera Models """ import base64 +import importlib import math +import os from dataclasses import dataclass +from distutils.util import strtobool from enum import Enum, auto from typing import Dict, List, Optional, Tuple, Union import cv2 import torch import torchvision +from torch.nn import Parameter from torchtyping import TensorType +import nerfstudio.utils.math import nerfstudio.utils.poses as pose_utils from nerfstudio.cameras import camera_utils from nerfstudio.cameras.rays import RayBundle +from nerfstudio.data.scene_box import SceneBox from nerfstudio.utils.tensor_dataclass import TensorDataclass @@ -308,6 +314,7 @@ class Cameras(TensorDataclass): distortion_params_delta: Optional[TensorType["num_rays":..., 6]] = None, keep_shape: Optional[bool] = None, disable_distortion: bool = False, + aabb_box: SceneBox = None, ) -> RayBundle: """Generates rays for the given camera indices. @@ -350,6 +357,7 @@ class Cameras(TensorDataclass): keeping dimensions. If False, we flatten at the end. If True, then we keep the shape of the camera_indices and coords tensors (if we can). disable_distortion: If True, disables distortion. + aabb_box: if not None will calculate nears and fars of the ray according to aabb box intesection Returns: Rays for the given camera indices and coords. @@ -449,6 +457,31 @@ class Cameras(TensorDataclass): if keep_shape is False: raybundle = raybundle.flatten() + if aabb_box: + with torch.no_grad(): + tensor_aabb = Parameter(aabb_box.aabb.flatten(), requires_grad=False) + + rays_o = raybundle.origins.contiguous() + rays_d = raybundle.directions.contiguous() + + tensor_aabb = tensor_aabb.to(rays_o.device) + shape = rays_o.shape + + rays_o = rays_o.reshape((-1, 3)) + rays_d = rays_d.reshape((-1, 3)) + + if strtobool(os.environ.get("INTERSECT_WITH_NERFACC", "TRUE")): + nerfacc = importlib.import_module("nerfacc") + t_min, t_max = nerfacc.ray_aabb_intersect(rays_o, rays_d, tensor_aabb) + else: + t_min, t_max = nerfstudio.utils.math.intersect_aabb(rays_o, rays_d, tensor_aabb) + + t_min = t_min.reshape([shape[0], shape[1], 1]) + t_max = t_max.reshape([shape[0], shape[1], 1]) + + raybundle.nears = t_min + raybundle.fars = t_max + # TODO: We should have to squeeze the last dimension here if we started with zero batch dims, but never have to, # so there might be a rogue squeeze happening somewhere, and this may cause some unintended behaviour # that we haven't caught yet with tests diff --git a/nerfstudio/model_components/ray_samplers.py b/nerfstudio/model_components/ray_samplers.py index eef905101..e41416053 100644 --- a/nerfstudio/model_components/ray_samplers.py +++ b/nerfstudio/model_components/ray_samplers.py @@ -449,6 +449,15 @@ class VolumetricSampler(Sampler): rays_o = ray_bundle.origins.contiguous() rays_d = ray_bundle.directions.contiguous() + + if ray_bundle.nears is not None and ray_bundle.fars is not None: + t_min = ray_bundle.nears.contiguous().reshape(-1) + t_max = ray_bundle.fars.contiguous().reshape(-1) + + else: + t_min = None + t_max = None + if ray_bundle.camera_indices is not None: camera_indices = ray_bundle.camera_indices.contiguous() else: @@ -457,6 +466,8 @@ class VolumetricSampler(Sampler): ray_indices, starts, ends = nerfacc.ray_marching( rays_o=rays_o, rays_d=rays_d, + t_min=t_min, + t_max=t_max, scene_aabb=self.scene_aabb, grid=self.occupancy_grid, # this is a workaround - using density causes crash and damage quality. should be fixed diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 947b0ce63..9b48c22c8 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -26,8 +26,9 @@ Example: rgb = rgb_renderer(rgb=field_outputs[FieldHeadNames.RGB], weights=weights) """ +import contextlib import math -from typing import Optional, Union +from typing import Generator, Optional, Union import nerfacc import torch @@ -38,6 +39,20 @@ from typing_extensions import Literal from nerfstudio.cameras.rays import RaySamples from nerfstudio.utils.math import components_from_spherical_harmonics +BACKGROUND_MODE_OVERRIDE: Optional[Literal["last_sample", "black", "random"]] = None + + +@contextlib.contextmanager +def background_mode_override_context(mode: Literal["last_sample", "black"]) -> Generator[None, None, None]: + """Context manager for setting background mode.""" + global BACKGROUND_MODE_OVERRIDE # pylint: disable=global-statement + old_background_mode = BACKGROUND_MODE_OVERRIDE + try: + BACKGROUND_MODE_OVERRIDE = mode + yield + finally: + BACKGROUND_MODE_OVERRIDE = old_background_mode + class RGBRenderer(nn.Module): """Standard volumetic rendering. @@ -55,7 +70,7 @@ class RGBRenderer(nn.Module): cls, rgb: TensorType["bs":..., "num_samples", 3], weights: TensorType["bs":..., "num_samples", 1], - background_color: Union[Literal["random", "last_sample"], TensorType[3]] = "random", + background_color: Union[Literal["random", "black", "last_sample"], TensorType[3]] = "random", ray_indices: Optional[TensorType["num_samples"]] = None, num_rays: Optional[int] = None, ) -> TensorType["bs":..., 3]: @@ -81,10 +96,14 @@ class RGBRenderer(nn.Module): comp_rgb = torch.sum(weights * rgb, dim=-2) accumulated_weight = torch.sum(weights, dim=-2) + if BACKGROUND_MODE_OVERRIDE is not None: + background_color = BACKGROUND_MODE_OVERRIDE if background_color == "last_sample": background_color = rgb[..., -1, :] if background_color == "random": background_color = torch.rand_like(comp_rgb).to(rgb.device) + if background_color == "black": + background_color = torch.zeros_like(comp_rgb).to(rgb.device) assert isinstance(background_color, torch.Tensor) comp_rgb = comp_rgb + background_color.to(weights.device) * (1.0 - accumulated_weight) diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index 6be459343..3de53c153 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -74,6 +74,7 @@ class Model(nn.Module): super().__init__() self.config = config self.scene_box = scene_box + self.render_aabb = None # the box that we want to render - should be a subset of scene_box self.num_train_data = num_train_data self.kwargs = kwargs self.collider = None diff --git a/nerfstudio/utils/colormaps.py b/nerfstudio/utils/colormaps.py index 9d51606c7..18d7aa86f 100644 --- a/nerfstudio/utils/colormaps.py +++ b/nerfstudio/utils/colormaps.py @@ -36,6 +36,7 @@ def apply_colormap(image: TensorType["bs":..., 1], cmap="viridis") -> TensorType colormap = cm.get_cmap(cmap) colormap = torch.tensor(colormap.colors).to(image.device) # type: ignore + image = torch.nan_to_num(image, 0) image_long = (image * 255).long() image_long_min = torch.min(image_long) image_long_max = torch.max(image_long) diff --git a/nerfstudio/utils/math.py b/nerfstudio/utils/math.py index bca98ee83..2658b1c39 100644 --- a/nerfstudio/utils/math.py +++ b/nerfstudio/utils/math.py @@ -191,3 +191,49 @@ def expected_sin(x_means: torch.Tensor, x_vars: torch.Tensor) -> torch.Tensor: """ return torch.exp(-0.5 * x_vars) * torch.sin(x_means) + + +def intersect_aabb(origins, directions, aabb): + """ + pytorch implementation of ray intersection with AABB box + :param origin: N,3 tensor of 3d positions + :param direction: N,3 tensor of normalized directions + :param aabb: 6,1 array of aabb box in the form of [x_min, y_min, z_min, x_max, y_max, z_max] + :return: t_min, t_max - two tensors of shapes N,1 representing the intersection signed distance from + the origin. t_min is clipped to 0 in case it is negative, in case of no intersection t_min is 1e10. + """ + + max_bound = 1e10 + + tx_min = (aabb[0] - origins[:, 0]) / directions[:, 0] + tx_max = (aabb[3] - origins[:, 0]) / directions[:, 0] + t_min = torch.min(tx_min, tx_max) + t_max = torch.max(tx_min, tx_max) + + ty_min_temp = (aabb[1] - origins[:, 1]) / directions[:, 1] + ty_max_temp = (aabb[4] - origins[:, 1]) / directions[:, 1] + ty_min = torch.min(ty_min_temp, ty_max_temp) + ty_max = torch.max(ty_min_temp, ty_max_temp) + + cond = torch.logical_or((t_min > ty_max), (ty_min > t_max)) + t_min = torch.where(cond, max_bound, t_min) + t_max = torch.where(cond, max_bound, t_max) + + t_min = torch.max(t_min, ty_min) + t_max = torch.min(t_max, ty_max) + + tz_min_temp = (aabb[2] - origins[:, 2]) / directions[:, 2] + tz_max_temp = (aabb[5] - origins[:, 2]) / directions[:, 2] + tz_min = torch.min(tz_min_temp, tz_max_temp) + tz_max = torch.max(tz_min_temp, tz_max_temp) + + cond = torch.logical_or((t_min > tz_max), tz_min > t_max) + t_min = torch.where(cond, max_bound, t_min) + t_max = torch.where(cond, max_bound, t_max) + + t_min = torch.max(t_min, tz_min) + t_max = torch.min(t_max, tz_max) + t_min = torch.where(t_min > 0, t_min, 0) + t_max = torch.where(t_min == max_bound, max_bound, t_max) + + return t_min, t_max diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index ccaa2fa1c..848e1dddc 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-01-19-0", + "version": "23-01-20-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx index 2b03f2cc1..d0294b191 100644 --- a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx @@ -27,10 +27,17 @@ export function RenderControls() { const target_train_util = useSelector( (state) => state.renderingState.targetTrainUtil, ); - const render_time = useSelector( - (state) => state.renderingState.renderTime, + const render_time = useSelector((state) => state.renderingState.renderTime); + const crop_enabled = useSelector( + (state) => state.renderingState.crop_enabled, ); + const box_size = useSelector((state) => state.renderingState.box_size); + + const crop_scale = useSelector((state) => state.renderingState.crop_scale); + + const crop_center = useSelector((state) => state.renderingState.crop_center); + const dispatch = useDispatch(); const set_max_resolution = (value) => { @@ -111,13 +118,13 @@ export function RenderControls() { const [display_render_time, set_display_render_time] = useState(false); - const receive_temporal_dist = e => { - const msg = msgpack.decode(new Uint8Array(e.data)); - if (msg.path === "/model/has_temporal_distortion") { - set_display_render_time(msg.data === "true"); - websocket.removeEventListener("message", receive_temporal_dist); - } - } + const receive_temporal_dist = (e) => { + const msg = msgpack.decode(new Uint8Array(e.data)); + if (msg.path === '/model/has_temporal_distortion') { + set_display_render_time(msg.data === 'true'); + websocket.removeEventListener('message', receive_temporal_dist); + } + }; websocket.addEventListener('message', receive_temporal_dist); const set_render_time = (value) => { @@ -139,6 +146,63 @@ export function RenderControls() { } }; + const set_crop_enabled = (value) => { + if (websocket.readyState === WebSocket.OPEN) { + dispatch({ + type: 'write', + path: 'renderingState/crop_enabled', + data: value, + }); + const cmd = 'write'; + const path = 'renderingState/crop_enabled'; + const data = { + type: cmd, + path, + data: value, + }; + const message = msgpack.encode(data); + websocket.send(message); + } + }; + + const set_crop_scale = (value) => { + if (websocket.readyState === WebSocket.OPEN) { + dispatch({ + type: 'write', + path: 'renderingState/crop_scale', + data: value, + }); + const cmd = 'write'; + const path = 'renderingState/crop_scale'; + const data = { + type: cmd, + path, + data: value, + }; + const message = msgpack.encode(data); + websocket.send(message); + } + }; + + const set_crop_center = (value) => { + if (websocket.readyState === WebSocket.OPEN) { + dispatch({ + type: 'write', + path: 'renderingState/crop_center', + data: value, + }); + const cmd = 'write'; + const path = 'renderingState/crop_center'; + const data = { + type: cmd, + path, + data: value, + }; + const message = msgpack.encode(data); + websocket.send(message); + } + }; + const [, setControls] = useControls( () => ({ // training speed @@ -197,17 +261,51 @@ export function RenderControls() { '1024px': () => setControls({ max_resolution: 1024 }), '2048px': () => setControls({ max_resolution: 2048 }), }), - // Dynamic NeRF rendering time - ...(display_render_time ? {render_time: { - label: 'Render Timestep', - value: render_time, - min: 0, - max: 1, - step: 0.01, - onChange: (v) => { - set_render_time(v); + // Enable Crop + crop_options: { + label: 'Crop Viewport', + value: crop_enabled, + onChange: (value) => { + set_crop_enabled(value); }, - }} : {}), + }, + crop_scale: { + label: '| Scale', + value: crop_scale, + min: 0, + max: 10, + step: 0.05, + render: (get) => get('crop_options'), + onChange: (v) => { + set_crop_scale(v); + }, + }, + crop_center: { + label: '| Center', + value: crop_center, + min: -10, + max: 10, + step: 0.05, + render: (get) => get('crop_options'), + onChange: (v) => { + set_crop_center(v); + }, + }, + // Dynamic NeRF rendering time + ...(display_render_time + ? { + render_time: { + label: 'Render Timestep', + value: render_time, + min: 0, + max: 1, + step: 0.01, + onChange: (v) => { + set_render_time(v); + }, + }, + } + : {}), }), [ outputOptions, @@ -215,6 +313,8 @@ export function RenderControls() { colormapOptions, colormapChoice, max_resolution, + crop_enabled, + box_size, target_train_util, render_time, display_render_time, diff --git a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx index 5218b82b8..d48edfff3 100644 --- a/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx +++ b/nerfstudio/viewer/app/src/modules/ViewerWindow/ViewerWindow.jsx @@ -18,9 +18,13 @@ const msgpack = require('msgpack-lite'); function CameraToggle() { const dispatch = useDispatch(); + const websocket = useContext(WebSocketContext).socket; const camera_choice = useSelector( (state) => state.renderingState.camera_choice, ); + const crop_enabled = useSelector( + (state) => state.renderingState.crop_enabled, + ); const set_camera_choice = (event: SelectChangeEvent, value: string[]) => { if (value != null) { dispatch({ @@ -28,6 +32,21 @@ function CameraToggle() { path: 'renderingState/camera_choice', data: value, }); + if (websocket.readyState === WebSocket.OPEN) { + const path = 'renderingState/crop_enabled'; + const data = { + type: 'write', + path, + data: false, + }; + if (value === 'Render Camera') { + data.data = false; + } else { + data.data = crop_enabled; + } + const message = msgpack.encode(data); + websocket.send(message); + } } }; diff --git a/nerfstudio/viewer/app/src/reducer.js b/nerfstudio/viewer/app/src/reducer.js index b9ebe42a8..7d59491cc 100644 --- a/nerfstudio/viewer/app/src/reducer.js +++ b/nerfstudio/viewer/app/src/reducer.js @@ -44,6 +44,11 @@ const initialState = { clipping_enabled: true, clipping_center: [0.0, 0.0, 0.0], clipping_box_scale: [2.0, 2.0, 2.0], + + // Crop Box Options + crop_enabled: false, + crop_scale: [2.0, 2.0, 2.0], + crop_center: [0.0, 0.0, 0.0], }, // the scene state sceneState: { diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index d7907c476..c6d8d38b2 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -42,6 +42,8 @@ from nerfstudio.cameras.cameras import Cameras, CameraType from nerfstudio.cameras.rays import RayBundle from nerfstudio.configs import base_config as cfg from nerfstudio.data.datasets.base_dataset import InputDataset +from nerfstudio.data.scene_box import SceneBox +from nerfstudio.model_components import renderers from nerfstudio.models.base_model import Model from nerfstudio.utils import colormaps, profiler, writer from nerfstudio.utils.decorators import check_main_thread, decorate_all @@ -144,8 +146,12 @@ class RenderThread(threading.Thread): outputs = None try: with SetTrace(self.state.check_interrupt): - with torch.no_grad(): - outputs = self.graph.get_outputs_for_camera_ray_bundle(self.camera_ray_bundle) + if self.state.prev_crop_enabled: + with renderers.background_mode_override_context("black"), torch.no_grad(): + outputs = self.graph.get_outputs_for_camera_ray_bundle(self.camera_ray_bundle) + else: + with torch.no_grad(): + outputs = self.graph.get_outputs_for_camera_ray_bundle(self.camera_ray_bundle) except Exception as e: # pylint: disable=broad-except self.exc = e @@ -217,6 +223,13 @@ class CheckThread(threading.Thread): self.state.check_interrupt_vis = True return + # check crop changes + crop_enabled = self.state.vis["renderingState/crop_enabled"].read() + if crop_enabled is not None: + if self.state.prev_crop_enabled != crop_enabled: + self.state.check_interrupt_vis = True + return + @decorate_all([check_main_thread]) class ViewerState: @@ -272,6 +285,9 @@ class ViewerState: self.moving_fps = 24 self.camera_moving = False self.prev_camera_timestamp = 0 + self.prev_crop_enabled = False + self.prev_crop_scale = None + self.prev_crop_center = None self.output_list = None @@ -335,6 +351,9 @@ class ViewerState: # set the initial state whether to train or not self.vis["renderingState/isTraining"].write(start_train) + max_scene_box = torch.max(dataset.scene_box.aabb[1] - dataset.scene_box.aabb[0]).item() + self.vis["renderingState/max_box_size"].write(max_scene_box) + # self.vis["renderingState/render_time"].write(str(0)) # set the properties of the camera @@ -381,6 +400,48 @@ class ViewerState: # remove the offer from the state tree self.vis["webrtc/offer"].delete() + def _update_render_aabb(self, graph): + """ + update the render aabb box for the viewer: + + :param graph: + :return: + """ + + crop_enabled = self.vis["renderingState/crop_enabled"].read() + if crop_enabled != self.prev_crop_enabled: + self.camera_moving = True + self.prev_crop_enabled = crop_enabled + self.prev_crop_scale = None + self.prev_crop_center = None + + if crop_enabled: + crop_scale = self.vis["renderingState/crop_scale"].read() + crop_center = self.vis["renderingState/crop_center"].read() + + if crop_scale != self.prev_crop_scale or crop_center != self.prev_crop_center: + self.camera_moving = True + self.prev_crop_scale = crop_scale + self.prev_crop_center = crop_center + + crop_scale = torch.tensor(crop_scale) + crop_center = torch.tensor(crop_center) + + box_min = crop_center - crop_scale / 2.0 + box_max = crop_center + crop_scale / 2.0 + + if isinstance(graph.render_aabb, SceneBox): + graph.render_aabb.aabb[0] = box_min + graph.render_aabb.aabb[1] = box_max + else: + graph.render_aabb = SceneBox(aabb=torch.stack([box_min, box_max], dim=0)) + + # maybe should update only if true change ? + json_ = graph.render_aabb.to_json() + self.vis["sceneState/sceneBox"].write(json_) + else: + graph.render_aabb = None + def update_scene(self, trainer, step: int, graph: Model, num_rays_per_batch: int) -> None: """updates the scene based on the graph weights @@ -481,6 +542,16 @@ class ViewerState: if self.prev_colormap_type != colormap_type: self.camera_moving = True + crop_scale = self.vis["renderingState/crop_scale"].read() + if self.prev_crop_enabled: + if self.prev_crop_scale != crop_scale: + self.camera_moving = True + + crop_center = self.vis["renderingState/crop_center"].read() + if self.prev_crop_enabled: + if self.prev_crop_center != crop_center: + self.camera_moving = True + return camera_object def _apply_colormap(self, outputs: Dict[str, Any], colors: torch.Tensor = None, eps=1e-6): @@ -760,6 +831,15 @@ class ViewerState: colormap_type = ColormapTypes.INIT if colormap_type is None else colormap_type self.prev_colormap_type = colormap_type + # update render aabb + try: + self._update_render_aabb(graph) + except RuntimeError as e: + self.vis["renderingState/log_errors"].write("Got an Error while trying to update aabb crop") + print(f"Error: {e}") + + time.sleep(0.5) # sleep to allow buffer to reset + # Calculate camera pose and intrinsics try: image_height, image_width = self._calculate_image_res(camera_object, is_training) @@ -811,7 +891,7 @@ class ViewerState: ) camera = camera.to(graph.device) - camera_ray_bundle = camera.generate_rays(camera_indices=0) + camera_ray_bundle = camera.generate_rays(camera_indices=0, aabb_box=graph.render_aabb) graph.eval() diff --git a/tests/utils/test_aabb_intersection.py b/tests/utils/test_aabb_intersection.py new file mode 100644 index 000000000..1e01dae0b --- /dev/null +++ b/tests/utils/test_aabb_intersection.py @@ -0,0 +1,178 @@ +""" +Test AABB intersection +""" +import importlib +import math +import os +import random +from distutils.util import strtobool + +import torch + +from nerfstudio.utils.math import intersect_aabb + +test_device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +def _swap_minmax(x, y): + if x < y: + return x, y + + return y, x + + +def _get_random_aabb_box(max_box, device): + """ + generate random aabb box + :param max_box: + :param device: + :return: + """ + x_min = random.uniform(-max_box, max_box) + x_max = random.uniform(-max_box, max_box) + + x_min, x_max = _swap_minmax(x_min, x_max) + + y_min = random.uniform(-max_box, max_box) + y_max = random.uniform(-max_box, max_box) + + y_min, y_max = _swap_minmax(y_min, y_max) + + z_min = random.uniform(-max_box, max_box) + z_max = random.uniform(-max_box, max_box) + + z_min, z_max = _swap_minmax(z_min, z_max) + + return torch.tensor([x_min, y_min, z_min, x_max, y_max, z_max], dtype=torch.float32, device=device) + + +def _get_random_rays(num_rays, max_box, aabb, device): + """ + genrate random rays that distribute in the general direction of the box. + The implementation is naive for now. + :param num_rays: + :param max_box: + :param aabb: + :param device: + :return: + """ + origins = [] + directions = [] + aabb_center = torch.tensor( + [0.5 * (aabb[0] + aabb[3]), 0.5 * (aabb[1] + aabb[4]), 0.5 * (aabb[2] + aabb[5])], + dtype=torch.float32, + device=device, + ) + + for _ in range(num_rays): + x = random.uniform(-max_box, max_box) + y = random.uniform(-max_box, max_box) + z = random.uniform(-max_box, max_box) + + origin = torch.tensor([x, y, z], dtype=torch.float32, device=device) + dirs_to_cent = aabb_center - origin + dirs_to_cent = dirs_to_cent / torch.linalg.norm(dirs_to_cent) + phi = math.asin(dirs_to_cent[2]) + theta = torch.atan2(dirs_to_cent[1], dirs_to_cent[0]) + + error_ang_phi = random.gauss(0, 0.05) + error_ang_theta = random.gauss(0, 0.1) + + phi = phi + error_ang_phi + theta = theta + error_ang_theta + + x_dir = math.cos(phi) * math.cos(theta) + y_dir = math.cos(phi) * math.sin(theta) + z_dir = math.sin(phi) + + origins.append([x, y, z]) + directions.append([x_dir, y_dir, z_dir]) + + origins = torch.tensor(origins, dtype=torch.float32, device=device) + directions = torch.tensor(directions, dtype=torch.float32, device=device) + + return origins, directions + + +def _is_point_on_aabb_boundary(point, aabb): + # check + error_threshold = 0.001 + for dim in range(3): + if point[dim] < (aabb[dim] - error_threshold) or point[dim] > (aabb[dim + 3] + error_threshold): + return False + # check that the points is near the boundary + for side in range(2): + for dim in range(3): + if abs(point[dim] - aabb[side * 3 + dim]) < error_threshold: + return True + return False + + +def _assert_points_on_aabb_boundary(points, aabb): + for point in points: + assert _is_point_on_aabb_boundary(point, aabb) + + +def test_rays_intersection_are_on_aabb_boundary(): + """ + test that org + dir*t_min and org + dir*t_max are on aabb boundary + :return: + """ + random.seed(496) + max_box_size = 100 + aabb = _get_random_aabb_box(max_box_size, test_device) + num_rays = 500 + max_test_size = 1000 + origins, directions = _get_random_rays(num_rays, max_test_size, aabb, test_device) + + t_min, t_max = intersect_aabb(origins, directions, aabb) + + mask = t_min < 1e10 + t_min = t_min[mask] + t_max = t_max[mask] + origins = origins[mask] + directions = directions[mask] + + # assert that we don't do something trivial + assert len(t_min) > 10 + + # assert that tmax is larger than t_min + assert torch.all(t_max >= t_min) + + positions_min = origins + directions * t_min.view(-1, 1) + positions_max = origins + directions * t_max.view(-1, 1) + + _assert_points_on_aabb_boundary(positions_min, aabb) + _assert_points_on_aabb_boundary(positions_max, aabb) + + +def test_equall_nerfacc(): + """ + test that the output of intersect_aabb is close to nerfacc.intersect_aabb + :return: + """ + check_nerf_acc = strtobool(os.environ.get("INTERSECT_WITH_NERFACC", "FALSE")) + if check_nerf_acc: + nerfacc = importlib.import_module("nerfacc") + random.seed(8128) + + max_box_size = 100 + aabb = _get_random_aabb_box(max_box_size, test_device) + num_rays = 500 + max_test_size = 1000 + + origins, directions = _get_random_rays(num_rays, max_test_size, aabb, test_device) + + # time1 = time.time() + t_min, t_max = intersect_aabb(origins, directions, aabb) + # time2 = time.time() + + # time3 = time.time() + t_min_nerfacc, t_max_nerfacc = nerfacc.ray_aabb_intersect(origins, directions, aabb) + # time4 = time.time() + + # print("pytorch ", time2-time1) + # print("nerfacc ", time4-time3) + + assert torch.allclose(t_min, t_min_nerfacc, rtol=0.001) + assert torch.allclose(t_max, t_max_nerfacc, rtol=0.001)