Allow users to modify background color when cropping (#1289)

This commit is contained in:
Matthew Tancik authored and GitHub committed 2023-01-25 17:47:45 -08:00
1 parent 80ae933811
commit 90489b5a6d
6 files changed
+51 -12

No files matched your search

+1
View File
@@ -134,6 +134,7 @@ class RaySamples(TensorDataclass):
transmittance = torch.exp(-transmittance) # [..., "num_samples"]
weights = alphas * transmittance # [..., "num_samples"]
weights = torch.nan_to_num(weights)
return weights
+14 -8
View File
@@ -39,19 +39,19 @@ from typing_extensions import Literal
from nerfstudio.cameras.rays import RaySamples
from nerfstudio.utils.math import components_from_spherical_harmonics
BACKGROUND_MODE_OVERRIDE: Optional[Literal["last_sample", "black", "random"]] = None
BACKGROUND_COLOR_OVERRIDE: Optional[TensorType[3]] = None
@contextlib.contextmanager
def background_mode_override_context(mode: Literal["last_sample", "black"]) -> Generator[None, None, None]:
def background_color_override_context(mode: TensorType[3]) -> Generator[None, None, None]:
"""Context manager for setting background mode."""
global BACKGROUND_MODE_OVERRIDE # pylint: disable=global-statement
old_background_mode = BACKGROUND_MODE_OVERRIDE
global BACKGROUND_COLOR_OVERRIDE # pylint: disable=global-statement
old_background_color = BACKGROUND_COLOR_OVERRIDE
try:
BACKGROUND_MODE_OVERRIDE = mode
BACKGROUND_COLOR_OVERRIDE = mode
yield
finally:
BACKGROUND_MODE_OVERRIDE = old_background_mode
BACKGROUND_COLOR_OVERRIDE = old_background_color
class RGBRenderer(nn.Module):
@@ -96,8 +96,8 @@ class RGBRenderer(nn.Module):
comp_rgb = torch.sum(weights * rgb, dim=-2)
accumulated_weight = torch.sum(weights, dim=-2)
if BACKGROUND_MODE_OVERRIDE is not None:
background_color = BACKGROUND_MODE_OVERRIDE
if BACKGROUND_COLOR_OVERRIDE is not None:
background_color = BACKGROUND_COLOR_OVERRIDE
if background_color == "last_sample":
background_color = rgb[..., -1, :]
if background_color == "random":
@@ -129,6 +129,8 @@ class RGBRenderer(nn.Module):
Outputs of rgb values.
"""
if not self.training:
rgb = torch.nan_to_num(rgb)
rgb = self.combine_rgb(
rgb, weights, background_color=self.background_color, ray_indices=ray_indices, num_rays=num_rays
)
@@ -182,7 +184,11 @@ class SHRenderer(nn.Module):
if self.activation is not None:
self.activation(rgb)
if not self.training:
rgb = torch.nan_to_num(rgb)
rgb = RGBRenderer.combine_rgb(rgb, weights, background_color=self.background_color)
if not self.training:
torch.clamp_(rgb, min=0.0, max=1.0)
return rgb
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "viewer",
"homepage": ".",
"version": "23-01-20-0",
"version": "23-01-25-0",
"private": true,
"dependencies": {
"@emotion/react": "^11.10.4",
@@ -48,7 +48,7 @@ export function RenderControls() {
(state) => state.renderingState.crop_enabled,
);
const box_size = useSelector((state) => state.renderingState.box_size);
const crop_bg_color = useSelector((state) => state.renderingState.crop_bg_color);
const crop_scale = useSelector((state) => state.renderingState.crop_scale);
@@ -167,6 +167,19 @@ export function RenderControls() {
);
},
},
crop_bg_color: {
label: '| Background Color',
value: crop_bg_color,
render: (get) => get('crop_options'),
onChange: (v) => {
dispatch_and_send(
websocket,
dispatch,
'renderingState/crop_bg_color',
v,
);
},
},
crop_scale: {
label: '| Scale',
value: crop_scale,
@@ -227,7 +240,6 @@ export function RenderControls() {
colormapChoice,
max_resolution,
crop_enabled,
box_size,
target_train_util,
render_time,
display_render_time,
+1
View File
@@ -47,6 +47,7 @@ const initialState = {
// Crop Box Options
crop_enabled: false,
crop_bg_color: {r: 38, g:42, b:55},
crop_scale: [2.0, 2.0, 2.0],
crop_center: [0.0, 0.0, 0.0],
},
+20 -1
View File
@@ -147,7 +147,14 @@ class RenderThread(threading.Thread):
try:
with SetTrace(self.state.check_interrupt):
if self.state.prev_crop_enabled:
with renderers.background_mode_override_context("black"), torch.no_grad():
color = self.state.prev_crop_bg_color
if color is None:
background_color = torch.tensor([0.0, 0.0, 0.0], device=self.graph.device)
else:
background_color = torch.tensor(
[color["r"] / 255.0, color["g"] / 255.0, color["b"] / 255.0], device=self.graph.device
)
with renderers.background_color_override_context(background_color), torch.no_grad():
outputs = self.graph.get_outputs_for_camera_ray_bundle(self.camera_ray_bundle)
else:
with torch.no_grad():
@@ -286,6 +293,7 @@ class ViewerState:
self.camera_moving = False
self.prev_camera_timestamp = 0
self.prev_crop_enabled = False
self.prev_crop_bg_color = None
self.prev_crop_scale = None
self.prev_crop_center = None
@@ -412,12 +420,18 @@ class ViewerState:
if crop_enabled != self.prev_crop_enabled:
self.camera_moving = True
self.prev_crop_enabled = crop_enabled
self.prev_crop_bg_color = None
self.prev_crop_scale = None
self.prev_crop_center = None
if crop_enabled:
crop_scale = self.vis["renderingState/crop_scale"].read()
crop_center = self.vis["renderingState/crop_center"].read()
crop_bg_color = self.vis["renderingState/crop_bg_color"].read()
if crop_bg_color != self.prev_crop_bg_color:
self.camera_moving = True
self.prev_crop_bg_color = crop_bg_color
if crop_scale != self.prev_crop_scale or crop_center != self.prev_crop_center:
self.camera_moving = True
@@ -542,6 +556,11 @@ class ViewerState:
if self.prev_colormap_type != colormap_type:
self.camera_moving = True
crop_bg_color = self.vis["renderingState/crop_bg_color"].read()
if self.prev_crop_enabled:
if self.prev_crop_bg_color != crop_bg_color:
self.camera_moving = True
crop_scale = self.vis["renderingState/crop_scale"].read()
if self.prev_crop_enabled:
if self.prev_crop_scale != crop_scale: