misc cleanup (#432)

* misc cuda improvements. added cone angle as param

* wandb as default. better organization for experiments

* turn off tensorboard

* disable eval when using viewer

* remove comment

* launch.json
This commit is contained in:
Ethan Weber authored and GitHub committed 2022-09-15 00:16:41 -07:00
1 parent ef8c4fbe23
commit 8973039e65
21 files changed
+230 -111

No files matched your search

+2 -2
View File
@@ -207,7 +207,7 @@ class Model(nn.Module):
return outputs
@abstractmethod
def log_test_image_outputs(self, image_idx, step, batch, outputs) -> float:
def log_test_image_outputs(self, image_idx, step, batch, outputs) -> Dict[str, float]:
"""Writes the test image outputs.
TODO: This shouldn't return a loss
@@ -218,7 +218,7 @@ class Model(nn.Module):
outputs: Outputs of the model.
Returns:
The psnr.
A dictionary of metrics.
"""
def load_model(self, loaded_state: Dict[str, Any]) -> None:
+1 -2
View File
@@ -106,7 +106,6 @@ class CompoundModel(Model):
param_groups["fields"] = list(self.field.parameters())
return param_groups
@torch.cuda.amp.autocast()
def get_outputs(self, ray_bundle: RayBundle):
# TODO(ruilongli)
# - train test difference
@@ -192,4 +191,4 @@ class CompoundModel(Model):
writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(psnr), step=step)
return psnr.item()
return {"psnr": float(psnr.item())}
+63 -30
View File
@@ -18,7 +18,8 @@ Implementation of Instant NGP.
from __future__ import annotations
from typing import Dict, List
from dataclasses import dataclass, field
from typing import Dict, List, Literal, Optional, Type
import torch
from torch.nn import Parameter
@@ -34,7 +35,7 @@ from nerfactory.fields.modules.field_heads import FieldHeadNames
from nerfactory.models.base import Model
from nerfactory.models.modules.ray_sampler import NGPSpacedSampler
from nerfactory.optimizers.loss import MSELoss
from nerfactory.utils import colors, misc, visualization, writer
from nerfactory.utils import colors, visualization, writer
from nerfactory.utils.callbacks import (
TrainingCallback,
TrainingCallbackAttributes,
@@ -42,6 +43,30 @@ from nerfactory.utils.callbacks import (
)
@dataclass
class InstantNGPModelConfig(cfg.ModelConfig):
"""Instant NGP Model Config"""
_target: Type = field(
default_factory=lambda: NGPModel
) # We can't write `NGPModel` directly, because `NGPModel` doesn't exist yet
"""target class to instantiate"""
enable_collider: bool = False
"""Whether to create a scene collider to filter rays."""
collider_params: Optional[Dict[str, float]] = None
"""Instant NGP doesn't use a collider."""
field_implementation: Literal["torch", "tcnn"] = "tcnn" # torch, tcnn, ...
"""one of "torch" or "tcnn", or other fields in 'field_implementation_to_class'"""
enable_density_field: bool = True
"""Whether to create a density field to filter samples."""
num_samples: int = 1024
"""Number of samples in field evaluation. Defaults to 1024,"""
cone_angle: float = 0.0
"""Should be set to 0.0 for blender scenes but 1./256 for real scenes."""
randomize_background: bool = False
"""Whether to randomize the background color. Defaults to False."""
class NGPModel(Model):
"""Instant NGP model
@@ -49,9 +74,9 @@ class NGPModel(Model):
config: instant NGP configuration to instantiate model
"""
config: cfg.InstantNGPModelConfig
config: InstantNGPModelConfig
def __init__(self, config: cfg.InstantNGPModelConfig, **kwargs) -> None:
def __init__(self, config: InstantNGPModelConfig, **kwargs) -> None:
assert config.field_implementation in field_implementation_to_class
self.field = None
super().__init__(config=config, **kwargs)
@@ -86,8 +111,6 @@ class NGPModel(Model):
self.ssim = structural_similarity_index_measure
self.lpips = LearnedPerceptualImagePatchSimilarity()
# no colliders default
def get_param_groups(self) -> Dict[str, List[Parameter]]:
param_groups = {}
if self.field is None:
@@ -95,17 +118,18 @@ class NGPModel(Model):
param_groups["fields"] = list(self.field.parameters())
return param_groups
@torch.cuda.amp.autocast()
def get_outputs(self, ray_bundle: RayBundle):
assert self.field is not None
# TODO(ruilongli)
# - train test difference
# - visualize "depth_density_grid"
num_rays = len(ray_bundle)
device = ray_bundle.origins.device
if self.field is None:
raise ValueError("populate_fields() must be called before get_outputs")
ray_samples, packed_info, t_min, t_max = self.sampler(ray_bundle, self.field.aabb)
ray_samples, packed_info, t_min, t_max = self.sampler(
ray_bundle, self.field.aabb, cone_angle=self.config.cone_angle
)
field_outputs = self.field.forward(ray_samples)
rgbs = field_outputs[FieldHeadNames.RGB]
@@ -127,7 +151,13 @@ class NGPModel(Model):
opacities,
)
accumulated_depth = torch.clip(accumulated_depth, t_min[:, None], t_max[:, None])
accumulated_color = accumulated_color + colors.WHITE.to(accumulated_color) * (1.0 - accumulated_weight)
if self.config.randomize_background:
background_colors = torch.rand((num_rays, 3), device=device) # (num_rays, 3)
else:
background_colors = colors.WHITE.to(accumulated_color) # (3,)
accumulated_color = accumulated_color + background_colors * (1.0 - accumulated_weight)
outputs = {
"rgb": accumulated_color,
@@ -138,33 +168,32 @@ class NGPModel(Model):
return outputs
def get_loss_dict(self, outputs, batch, metrics_dict, loss_coefficients):
device = self.device
image = batch["image"].to(device)
if "alive_ray_mask" in outputs:
mask = outputs["alive_ray_mask"]
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
else:
rgb_loss = self.rgb_loss(image, outputs["rgb"])
image = batch["image"].to(self.device)
mask = outputs["alive_ray_mask"]
rgb_loss = self.rgb_loss(image[mask], outputs["rgb"][mask])
loss_dict = {"rgb_loss": rgb_loss}
loss_dict = misc.scale_dict(loss_dict, loss_coefficients)
return loss_dict
def log_test_image_outputs(self, image_idx, step, batch, outputs):
image = batch["image"].to(outputs["rgb"].device)
image = batch["image"].to(self.device)
rgb = outputs["rgb"]
acc = visualization.apply_colormap(outputs["accumulation"])
depth = visualization.apply_depth_colormap(
outputs["depth"],
accumulation=outputs["accumulation"],
)
alive_ray_mask = visualization.apply_colormap(outputs["alive_ray_mask"])
combined_rgb = torch.cat([image, rgb], dim=1)
combined_acc = torch.cat([acc], dim=1)
combined_depth = torch.cat([depth], dim=1)
combined_alive_ray_mask = torch.cat([alive_ray_mask], dim=1)
writer.put_image(name=f"img/image_idx_{image_idx}", image=combined_rgb, step=step)
writer.put_image(name=f"accumulation/image_idx_{image_idx}", image=combined_acc, step=step)
writer.put_image(name=f"depth/image_idx_{image_idx}", image=combined_depth, step=step)
# TODO: return an image dictionary instead of logging here
writer.put_image(name="Eval Images/img", image=combined_rgb, step=step)
writer.put_image(name="Eval Images/accumulation", image=combined_acc, step=step)
writer.put_image(name="Eval Images/depth", image=combined_depth, step=step)
writer.put_image(name="Eval Images/alive_ray_mask", image=combined_alive_ray_mask, step=step)
# Switch images from [H, W, C] to [1, C, H, W] for metrics computations
image = torch.moveaxis(image, -1, 0)[None, ...]
@@ -174,10 +203,14 @@ class NGPModel(Model):
ssim = self.ssim(image, rgb)
lpips = self.lpips(image, rgb)
writer.put_scalar(name=f"psnr/val_{image_idx}-fine", scalar=float(psnr), step=step)
writer.put_scalar(name=f"ssim/val_{image_idx}", scalar=float(ssim), step=step) # type: ignore
writer.put_scalar(name=f"lpips/val_{image_idx}", scalar=float(lpips), step=step)
writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(psnr), step=step)
return psnr.item()
# all of these metrics will be logged as scalars
metrics_dict = {
"psnr": float(psnr.item()),
"image_idx": image_idx,
"ssim": float(ssim), # type: ignore
"lpips": float(lpips),
"acc_min": float(acc.min().item()),
"acc_max": float(acc.max().item()),
}
# TODO(ethan): return an image dictionary
return metrics_dict
+1 -1
View File
@@ -191,4 +191,4 @@ class MipNerfModel(Model):
writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(fine_psnr), step=step)
return fine_psnr.item()
return {"psnr": float(fine_psnr.item())}
+1 -1
View File
@@ -210,4 +210,4 @@ class MipNerf360Model(Model):
writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(fine_psnr), step=step)
return fine_psnr.item()
return {"psnr": float(fine_psnr.item())}
+4 -5
View File
@@ -360,6 +360,7 @@ class NGPSpacedSampler(Sampler):
ray_bundle: RayBundle,
aabb: TensorType[2, 3],
num_samples: Optional[int] = None,
cone_angle: Optional[float] = 0.0,
marching_steps: Optional[int] = 128,
t_min: Optional[TensorType["num_rays"]] = None,
t_max: Optional[TensorType["num_rays"]] = None,
@@ -373,6 +374,7 @@ class NGPSpacedSampler(Sampler):
ray_bundle: Rays to generate samples for
aabb: Bounding box of the scene.
num_samples: Number of samples per ray
cone_angle: 0.0 for nerf-synthetic and 1.0/256 for real scenes
Returns:
First return is ray samples in a packed way where only the valid samples are kept.
@@ -385,16 +387,13 @@ class NGPSpacedSampler(Sampler):
num_samples = num_samples or self.num_samples
aabb = aabb.flatten()
aabb = aabb.flatten().contiguous()
rays_o = ray_bundle.origins.contiguous()
rays_d = ray_bundle.directions.contiguous()
scene_scale = (aabb[3:6] - aabb[0:3]).max()
if t_min is None or t_max is None:
# TODO(ruilongli): this clipping here is stupid. Try to avoid that.
t_min, t_max = nerfactory_cuda.ray_aabb_intersect(rays_o, rays_d, aabb)
t_min = torch.clamp(t_min, max=1e10) # type: ignore
t_max = torch.clamp(t_max, max=1e10) # type: ignore
marching_steps = -1 # disable marching mode for now
# TODO(ruilongli): * 16 is for original impl for training. Need to run
@@ -418,7 +417,7 @@ class NGPSpacedSampler(Sampler):
marching_steps,
max_samples_per_batch,
num_samples,
0.0,
cone_angle,
scene_scale,
)
+6
View File
@@ -239,3 +239,9 @@ class NerfWModel(Model):
if "mask" in batch:
mask = batch["mask"].repeat(1, 1, 3)
writer.put_image(name=f"mask/image_idx_{image_idx}", image=mask, step=step)
# this doesn't really make sense but do it anyway
fine_psnr = self.psnr(image, rgb_fine)
return {
"psnr": float(fine_psnr.item()),
}
+3 -1
View File
@@ -226,4 +226,6 @@ class TensoRFModel(Model):
writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(psnr), step=step)
return psnr.item()
return {
"psnr": float(psnr.item()),
}
+1 -1
View File
@@ -222,4 +222,4 @@ class NeRFModel(Model):
writer.put_scalar(name=writer.EventName.CURR_TEST_PSNR, scalar=float(fine_psnr), step=step)
return fine_psnr.item()
return {"psnr": float(fine_psnr.item())}