diff --git a/nerfstudio/cameras/rays.py b/nerfstudio/cameras/rays.py index e81e52642..8718712fb 100644 --- a/nerfstudio/cameras/rays.py +++ b/nerfstudio/cameras/rays.py @@ -134,6 +134,7 @@ class RaySamples(TensorDataclass): transmittance = torch.exp(-transmittance) # [..., "num_samples"] weights = alphas * transmittance # [..., "num_samples"] + weights = torch.nan_to_num(weights) return weights diff --git a/nerfstudio/model_components/renderers.py b/nerfstudio/model_components/renderers.py index 9b48c22c8..24ce46085 100644 --- a/nerfstudio/model_components/renderers.py +++ b/nerfstudio/model_components/renderers.py @@ -39,19 +39,19 @@ from typing_extensions import Literal from nerfstudio.cameras.rays import RaySamples from nerfstudio.utils.math import components_from_spherical_harmonics -BACKGROUND_MODE_OVERRIDE: Optional[Literal["last_sample", "black", "random"]] = None +BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None @contextlib.contextmanager -def background_mode_override_context(mode: Literal["last_sample", "black"]) -> Generator[None, None, None]: +def background_color_override_context(mode: TensorType[3]) -> Generator[None, None, None]: """Context manager for setting background mode.""" - global BACKGROUND_MODE_OVERRIDE # pylint: disable=global-statement - old_background_mode = BACKGROUND_MODE_OVERRIDE + global BACKGROUND_COLOR_OVERRIDE # pylint: disable=global-statement + old_background_color = BACKGROUND_COLOR_OVERRIDE try: - BACKGROUND_MODE_OVERRIDE = mode + BACKGROUND_COLOR_OVERRIDE = mode yield finally: - BACKGROUND_MODE_OVERRIDE = old_background_mode + BACKGROUND_COLOR_OVERRIDE = old_background_color class RGBRenderer(nn.Module): @@ -96,8 +96,8 @@ class RGBRenderer(nn.Module): comp_rgb = torch.sum(weights * rgb, dim=-2) accumulated_weight = torch.sum(weights, dim=-2) - if BACKGROUND_MODE_OVERRIDE is not None: - background_color = BACKGROUND_MODE_OVERRIDE + if BACKGROUND_COLOR_OVERRIDE is not None: + background_color = BACKGROUND_COLOR_OVERRIDE if background_color == "last_sample": background_color = rgb[..., -1, :] if background_color == "random": @@ -129,6 +129,8 @@ class RGBRenderer(nn.Module): Outputs of rgb values. """ + if not self.training: + rgb = torch.nan_to_num(rgb) rgb = self.combine_rgb( rgb, weights, background_color=self.background_color, ray_indices=ray_indices, num_rays=num_rays ) @@ -182,7 +184,11 @@ class SHRenderer(nn.Module): if self.activation is not None: self.activation(rgb) + if not self.training: + rgb = torch.nan_to_num(rgb) rgb = RGBRenderer.combine_rgb(rgb, weights, background_color=self.background_color) + if not self.training: + torch.clamp_(rgb, min=0.0, max=1.0) return rgb diff --git a/nerfstudio/viewer/app/package.json b/nerfstudio/viewer/app/package.json index 848e1dddc..37f2b7bbb 100644 --- a/nerfstudio/viewer/app/package.json +++ b/nerfstudio/viewer/app/package.json @@ -1,7 +1,7 @@ { "name": "viewer", "homepage": ".", - "version": "23-01-20-0", + "version": "23-01-25-0", "private": true, "dependencies": { "@emotion/react": "^11.10.4", diff --git a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx index e710b6733..44e3f5ea0 100644 --- a/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx +++ b/nerfstudio/viewer/app/src/modules/ConfigPanel/ConfigPanel.jsx @@ -48,7 +48,7 @@ export function RenderControls() { (state) => state.renderingState.crop_enabled, ); - const box_size = useSelector((state) => state.renderingState.box_size); + const crop_bg_color = useSelector((state) => state.renderingState.crop_bg_color); const crop_scale = useSelector((state) => state.renderingState.crop_scale); @@ -167,6 +167,19 @@ export function RenderControls() { ); }, }, + crop_bg_color: { + label: '| Background Color', + value: crop_bg_color, + render: (get) => get('crop_options'), + onChange: (v) => { + dispatch_and_send( + websocket, + dispatch, + 'renderingState/crop_bg_color', + v, + ); + }, + }, crop_scale: { label: '| Scale', value: crop_scale, @@ -227,7 +240,6 @@ export function RenderControls() { colormapChoice, max_resolution, crop_enabled, - box_size, target_train_util, render_time, display_render_time, diff --git a/nerfstudio/viewer/app/src/reducer.js b/nerfstudio/viewer/app/src/reducer.js index 7d59491cc..0ee2dd2ed 100644 --- a/nerfstudio/viewer/app/src/reducer.js +++ b/nerfstudio/viewer/app/src/reducer.js @@ -47,6 +47,7 @@ const initialState = { // Crop Box Options crop_enabled: false, + crop_bg_color: {r: 38, g:42, b:55}, crop_scale: [2.0, 2.0, 2.0], crop_center: [0.0, 0.0, 0.0], }, diff --git a/nerfstudio/viewer/server/viewer_utils.py b/nerfstudio/viewer/server/viewer_utils.py index c6d8d38b2..2ac82273d 100644 --- a/nerfstudio/viewer/server/viewer_utils.py +++ b/nerfstudio/viewer/server/viewer_utils.py @@ -147,7 +147,14 @@ class RenderThread(threading.Thread): try: with SetTrace(self.state.check_interrupt): if self.state.prev_crop_enabled: - with renderers.background_mode_override_context("black"), torch.no_grad(): + color = self.state.prev_crop_bg_color + if color is None: + background_color = torch.tensor([0.0, 0.0, 0.0], device=self.graph.device) + else: + background_color = torch.tensor( + [color["r"] / 255.0, color["g"] / 255.0, color["b"] / 255.0], device=self.graph.device + ) + with renderers.background_color_override_context(background_color), torch.no_grad(): outputs = self.graph.get_outputs_for_camera_ray_bundle(self.camera_ray_bundle) else: with torch.no_grad(): @@ -286,6 +293,7 @@ class ViewerState: self.camera_moving = False self.prev_camera_timestamp = 0 self.prev_crop_enabled = False + self.prev_crop_bg_color = None self.prev_crop_scale = None self.prev_crop_center = None @@ -412,12 +420,18 @@ class ViewerState: if crop_enabled != self.prev_crop_enabled: self.camera_moving = True self.prev_crop_enabled = crop_enabled + self.prev_crop_bg_color = None self.prev_crop_scale = None self.prev_crop_center = None if crop_enabled: crop_scale = self.vis["renderingState/crop_scale"].read() crop_center = self.vis["renderingState/crop_center"].read() + crop_bg_color = self.vis["renderingState/crop_bg_color"].read() + + if crop_bg_color != self.prev_crop_bg_color: + self.camera_moving = True + self.prev_crop_bg_color = crop_bg_color if crop_scale != self.prev_crop_scale or crop_center != self.prev_crop_center: self.camera_moving = True @@ -542,6 +556,11 @@ class ViewerState: if self.prev_colormap_type != colormap_type: self.camera_moving = True + crop_bg_color = self.vis["renderingState/crop_bg_color"].read() + if self.prev_crop_enabled: + if self.prev_crop_bg_color != crop_bg_color: + self.camera_moving = True + crop_scale = self.vis["renderingState/crop_scale"].read() if self.prev_crop_enabled: if self.prev_crop_scale != crop_scale: