mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 14:45:38 +08:00
1 parent
5924802f5c
commit
fae01d3ca6
22 files changed
+17
-1887
No files matched your search
@@ -31,7 +31,6 @@ from nerfactory.cameras.rays import RayBundle
|
||||
from nerfactory.configs.base import InstantiateConfig
|
||||
from nerfactory.configs.utils import to_immutable_dict
|
||||
from nerfactory.datamanagers.structs import SceneBounds
|
||||
from nerfactory.fields.density_fields.density_grid import DensityGrid
|
||||
from nerfactory.models.modules.scene_colliders import NearFarCollider
|
||||
from nerfactory.utils.callbacks import TrainingCallback, TrainingCallbackAttributes
|
||||
|
||||
@@ -48,18 +47,6 @@ class ModelConfig(InstantiateConfig):
|
||||
collider_params: Optional[Dict[str, float]] = to_immutable_dict({"near_plane": 2.0, "far_plane": 6.0})
|
||||
"""parameters to instantiate scene collider with"""
|
||||
loss_coefficients: Dict[str, float] = to_immutable_dict({"rgb_loss_coarse": 1.0, "rgb_loss_fine": 1.0})
|
||||
"""Loss specific weights."""
|
||||
enable_density_field: bool = False
|
||||
"""Whether to create a density field to filter samples."""
|
||||
density_field_params: Dict[str, Any] = to_immutable_dict(
|
||||
{
|
||||
"center": 0.0, # simply set it as the center of the scene bbox
|
||||
"base_scale": 3.0, # simply set it as the scale of the scene bbox
|
||||
"num_cascades": 1, # if using more than 1 cascade, the `base_scale` can be smaller than scene scale.
|
||||
"resolution": 128,
|
||||
"update_every_num_iters": 16,
|
||||
}
|
||||
)
|
||||
"""parameters to instantiate density field with"""
|
||||
eval_num_rays_per_chunk: int = 4096
|
||||
"""specifies number of rays per chunk during eval"""
|
||||
@@ -88,7 +75,6 @@ class Model(nn.Module):
|
||||
self.config = config
|
||||
self.scene_bounds = scene_bounds
|
||||
self.num_train_data = num_train_data
|
||||
self.density_field = None
|
||||
self.kwargs = kwargs
|
||||
self.collider = None
|
||||
self.populate_modules() # populate the modules
|
||||
@@ -112,14 +98,6 @@ class Model(nn.Module):
|
||||
# default instantiates optional modules that are common among many networks
|
||||
# NOTE: call `super().populate_modules()` in subclasses
|
||||
|
||||
if self.config.enable_density_field:
|
||||
self.density_field = DensityGrid(
|
||||
center=self.config.density_field_params["center"],
|
||||
base_scale=self.config.density_field_params["base_scale"],
|
||||
num_cascades=self.config.density_field_params["num_cascades"],
|
||||
resolution=self.config.density_field_params["resolution"],
|
||||
update_every_num_iters=self.config.density_field_params["update_every_num_iters"],
|
||||
)
|
||||
if self.config.enable_collider:
|
||||
self.collider = NearFarCollider(
|
||||
near_plane=self.config.collider_params["near_plane"], far_plane=self.config.collider_params["far_plane"]
|
||||
|
||||
@@ -26,7 +26,6 @@ from torch import nn
|
||||
from torchtyping import TensorType
|
||||
|
||||
from nerfactory.cameras.rays import Frustums, RayBundle, RaySamples
|
||||
from nerfactory.fields.density_fields.density_grid import DensityGrid
|
||||
|
||||
|
||||
class Sampler(nn.Module):
|
||||
@@ -34,34 +33,22 @@ class Sampler(nn.Module):
|
||||
|
||||
Args:
|
||||
num_samples: number of samples to take
|
||||
density_field: density grid specifying weighting of
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_samples: Optional[int] = None,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 0.01,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_samples = num_samples
|
||||
self.density_field = density_field
|
||||
self.weight_threshold = weight_threshold
|
||||
|
||||
@abstractmethod
|
||||
def generate_ray_samples(self) -> RaySamples:
|
||||
"""Generate Ray Samples"""
|
||||
|
||||
def forward(self, *args, **kwargs) -> RaySamples:
|
||||
"""Generate ray samples with optional density filtering"""
|
||||
ray_samples = self.generate_ray_samples(*args, **kwargs)
|
||||
if self.density_field is not None:
|
||||
densities = self.density_field.get_densities(ray_samples)
|
||||
deltas = torch.clamp(ray_samples.frustums.ends - ray_samples.frustums.starts, min=1e-10)
|
||||
density_threshold = torch.clamp(self.weight_threshold / deltas, max=self.density_field.mean_density)
|
||||
valid_mask = densities > density_threshold
|
||||
ray_samples.set_valid_mask(valid_mask & ray_samples.valid_mask)
|
||||
return ray_samples
|
||||
"""Generate ray samples"""
|
||||
return self.generate_ray_samples(*args, **kwargs)
|
||||
|
||||
|
||||
class SpacedSampler(Sampler):
|
||||
@@ -72,8 +59,6 @@ class SpacedSampler(Sampler):
|
||||
spacing_fn: Function that dictates sample spacing (ie `lambda x : x` is uniform).
|
||||
spacing_fn_inv: The inverse of spacing_fn.
|
||||
train_stratified: Use stratified sampling during training. Defults to True
|
||||
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
|
||||
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -83,10 +68,8 @@ class SpacedSampler(Sampler):
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
super().__init__(num_samples=num_samples, density_field=density_field, weight_threshold=weight_threshold)
|
||||
super().__init__(num_samples=num_samples)
|
||||
self.train_stratified = train_stratified
|
||||
self.single_jitter = single_jitter
|
||||
self.spacing_fn = spacing_fn
|
||||
@@ -149,8 +132,6 @@ class UniformSampler(SpacedSampler):
|
||||
Args:
|
||||
num_samples: Number of samples per ray
|
||||
train_stratified: Use stratified sampling during training. Defults to True
|
||||
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
|
||||
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -158,8 +139,6 @@ class UniformSampler(SpacedSampler):
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
@@ -167,8 +146,6 @@ class UniformSampler(SpacedSampler):
|
||||
spacing_fn_inv=lambda x: x,
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
|
||||
@@ -178,8 +155,6 @@ class LinearDisparitySampler(SpacedSampler):
|
||||
Args:
|
||||
num_samples: Number of samples per ray
|
||||
train_stratified: Use stratified sampling during training. Defults to True
|
||||
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
|
||||
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -187,8 +162,6 @@ class LinearDisparitySampler(SpacedSampler):
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
@@ -196,8 +169,6 @@ class LinearDisparitySampler(SpacedSampler):
|
||||
spacing_fn_inv=lambda x: 1 / x,
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
|
||||
@@ -207,8 +178,6 @@ class SqrtSampler(SpacedSampler):
|
||||
Args:
|
||||
num_samples: Number of samples per ray
|
||||
train_stratified: Use stratified sampling during training. Defults to True
|
||||
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
|
||||
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -216,8 +185,6 @@ class SqrtSampler(SpacedSampler):
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
@@ -225,8 +192,6 @@ class SqrtSampler(SpacedSampler):
|
||||
spacing_fn_inv=lambda x: x**2,
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
|
||||
@@ -236,8 +201,6 @@ class LogSampler(SpacedSampler):
|
||||
Args:
|
||||
num_samples: Number of samples per ray
|
||||
train_stratified: Use stratified sampling during training. Defults to True
|
||||
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
|
||||
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -245,8 +208,6 @@ class LogSampler(SpacedSampler):
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
@@ -254,8 +215,6 @@ class LogSampler(SpacedSampler):
|
||||
spacing_fn_inv=torch.exp,
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
|
||||
@@ -267,8 +226,6 @@ class UniformLinDispPiecewiseSampler(SpacedSampler):
|
||||
Args:
|
||||
num_samples: Number of samples per ray
|
||||
train_stratified: Use stratified sampling during training. Defults to True
|
||||
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
|
||||
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -276,8 +233,6 @@ class UniformLinDispPiecewiseSampler(SpacedSampler):
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified=True,
|
||||
single_jitter=False,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
num_samples=num_samples,
|
||||
@@ -285,8 +240,6 @@ class UniformLinDispPiecewiseSampler(SpacedSampler):
|
||||
spacing_fn_inv=lambda x: torch.where(x < 0.5, 2 * x, 1 / (2 - 2 * x)),
|
||||
train_stratified=train_stratified,
|
||||
single_jitter=single_jitter,
|
||||
density_field=density_field,
|
||||
weight_threshold=weight_threshold,
|
||||
)
|
||||
|
||||
|
||||
@@ -297,8 +250,6 @@ class PDFSampler(Sampler):
|
||||
num_samples: Number of samples per ray
|
||||
train_stratified: Randomize location within each bin during training.
|
||||
include_original: Add original samples to ray.
|
||||
density_field: Density grid. If provides, samples below weight_threshold as set as invalid.
|
||||
weight_threshold: Removes samples below threshold weight. Only used if density field is provided.
|
||||
histogram_padding: Amount to weights prior to computing PDF.
|
||||
"""
|
||||
|
||||
@@ -307,11 +258,9 @@ class PDFSampler(Sampler):
|
||||
num_samples: Optional[int] = None,
|
||||
train_stratified: bool = True,
|
||||
include_original: bool = True,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
histogram_padding: float = 0.01,
|
||||
) -> None:
|
||||
super().__init__(num_samples=num_samples, density_field=density_field, weight_threshold=weight_threshold)
|
||||
super().__init__(num_samples=num_samples)
|
||||
self.train_stratified = train_stratified
|
||||
self.include_original = include_original
|
||||
self.histogram_padding = histogram_padding
|
||||
@@ -490,9 +439,6 @@ class VolumetricSampler(Sampler):
|
||||
The ray_indices contains the indices of the rays that each sample belongs to.
|
||||
"""
|
||||
|
||||
if self.density_field is not None:
|
||||
raise ValueError("NGPSpacedSampler does not support a density field")
|
||||
|
||||
rays_o = ray_bundle.origins.contiguous()
|
||||
rays_d = ray_bundle.directions.contiguous()
|
||||
if ray_bundle.camera_indices is not None:
|
||||
@@ -550,10 +496,8 @@ class ProposalNetworkSampler(Sampler):
|
||||
num_nerf_samples_per_ray: int = 32,
|
||||
num_proposal_network_iterations: int = 2,
|
||||
single_jitter: bool = True,
|
||||
density_field: Optional[DensityGrid] = None,
|
||||
weight_threshold: float = 1e-2,
|
||||
) -> None:
|
||||
super().__init__(density_field=density_field, weight_threshold=weight_threshold)
|
||||
super().__init__()
|
||||
self.num_proposal_samples_per_ray = num_proposal_samples_per_ray
|
||||
self.num_nerf_samples_per_ray = num_nerf_samples_per_ray
|
||||
self.num_proposal_network_iterations = num_proposal_network_iterations
|
||||
|
||||
@@ -151,10 +151,8 @@ class TensoRFModel(Model):
|
||||
)
|
||||
|
||||
# samplers
|
||||
self.sampler_uniform = UniformSampler(
|
||||
num_samples=self.config.num_coarse_samples, density_field=self.density_field
|
||||
)
|
||||
self.sampler_pdf = PDFSampler(num_samples=self.config.num_importance_samples, density_field=self.density_field)
|
||||
self.sampler_uniform = UniformSampler(num_samples=self.config.num_coarse_samples)
|
||||
self.sampler_pdf = PDFSampler(num_samples=self.config.num_importance_samples)
|
||||
|
||||
# renderers
|
||||
self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
|
||||
|
||||
@@ -39,11 +39,6 @@ from nerfactory.renderers.renderers import (
|
||||
RGBRenderer,
|
||||
)
|
||||
from nerfactory.utils import colors, misc, visualization
|
||||
from nerfactory.utils.callbacks import (
|
||||
TrainingCallback,
|
||||
TrainingCallbackAttributes,
|
||||
TrainingCallbackLocation,
|
||||
)
|
||||
|
||||
|
||||
class NeRFModel(Model):
|
||||
@@ -66,23 +61,6 @@ class NeRFModel(Model):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_training_callbacks(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes # pylint: disable=unused-argument
|
||||
) -> List[TrainingCallback]:
|
||||
if self.field_coarse is None:
|
||||
raise ValueError("populate fields must be called before get_training_callbacks.")
|
||||
callbacks = []
|
||||
if self.density_field is not None:
|
||||
callbacks = [
|
||||
TrainingCallback(
|
||||
where_to_run=[TrainingCallbackLocation.AFTER_TRAIN_ITERATION],
|
||||
update_every_num_iters=self.density_field.update_every_num_iters,
|
||||
func=self.density_field.update_density_grid,
|
||||
kwargs={"density_eval_func": self.field_coarse.density_fn}, # type: ignore
|
||||
)
|
||||
]
|
||||
return callbacks
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules"""
|
||||
super().populate_modules()
|
||||
@@ -99,10 +77,8 @@ class NeRFModel(Model):
|
||||
self.field_fine = NeRFField(position_encoding=position_encoding, direction_encoding=direction_encoding)
|
||||
|
||||
# samplers
|
||||
self.sampler_uniform = UniformSampler(
|
||||
num_samples=self.config.num_coarse_samples, density_field=self.density_field
|
||||
)
|
||||
self.sampler_pdf = PDFSampler(num_samples=self.config.num_importance_samples, density_field=self.density_field)
|
||||
self.sampler_uniform = UniformSampler(num_samples=self.config.num_coarse_samples)
|
||||
self.sampler_pdf = PDFSampler(num_samples=self.config.num_importance_samples)
|
||||
|
||||
# renderers
|
||||
self.renderer_rgb = RGBRenderer(background_color=colors.WHITE)
|
||||
|
||||
Reference in new issue
Block a user