mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 06:14:03 +08:00
82 lines
2.9 KiB
Python
82 lines
2.9 KiB
Python
"""
|
|
Implementation of vanilla nerf.
|
|
"""
|
|
|
|
|
|
import logging
|
|
|
|
import torch
|
|
from omegaconf import DictConfig
|
|
from torch import nn
|
|
from torchtyping import TensorType
|
|
|
|
from mattport.nerf.dataset.utils import DatasetInputs, get_dataset_inputs
|
|
from mattport.nerf.field.nerf import NeRFField
|
|
from mattport.nerf.field_modules.encoding import NeRFEncoding
|
|
from mattport.nerf.field_modules.field_outputs import DensityFieldOutput, RGBFieldOutput
|
|
from mattport.nerf.field_modules.mlp import MLP
|
|
from mattport.nerf.field_modules.ray_generator import RayGenerator
|
|
from mattport.nerf.graph.base import Graph
|
|
from mattport.nerf.loss import MSELoss
|
|
from mattport.nerf.renderers import RGBRenderer
|
|
from mattport.nerf.sampler import UniformSampler
|
|
|
|
|
|
class NeRFGraph(Graph):
|
|
def __init__(self, intrinsics=None, camera_to_world=None) -> None:
|
|
super().__init__(intrinsics=intrinsics, camera_to_world=camera_to_world)
|
|
|
|
def populate_modules(self):
|
|
# ray generator
|
|
self.ray_generator = RayGenerator(self.intrinsics, self.camera_to_world)
|
|
|
|
# samplers
|
|
self.sampler_uniform = UniformSampler(near_plane=0.1, far_plane=4.0, num_samples=64)
|
|
self.sampler_pdf = UniformSampler(near_plane=0.1, far_plane=4.0, num_samples=64)
|
|
|
|
# field
|
|
self.field_coarse = NeRFField()
|
|
self.field_fine = NeRFField()
|
|
|
|
# renderers
|
|
self.renderer_rgb = RGBRenderer()
|
|
|
|
# losses
|
|
self.rgb_loss = MSELoss()
|
|
|
|
def forward(self, ray_indices: TensorType["num_rays", 3]):
|
|
"""Takes in the ray indices and renders out values."""
|
|
|
|
ray_bundle = self.ray_generator.forward(ray_indices)
|
|
ray_samples = self.sampler_uniform(ray_bundle)
|
|
|
|
xyz = ray_samples.get_positions(ray_bundle) # (num_rays, num_samples, 3)
|
|
num_rays, num_samples = xyz.shape[:2]
|
|
xyz = xyz.view(-1, 3)
|
|
dirs = (
|
|
ray_bundle.directions.unsqueeze(1).repeat(1, num_samples, 1).view(num_rays * num_samples, 3)
|
|
) # (num_rays, 3)
|
|
|
|
field_output_rgb, field_output_density = self.field_coarse(xyz=xyz, dirs=dirs)
|
|
# move these back to the correct shape
|
|
field_output_rgb = field_output_rgb.view(num_rays, num_samples, 3)
|
|
field_output_density = field_output_density.view(num_rays, num_samples)
|
|
|
|
deltas = ray_samples.get_deltas()
|
|
rgb_coarse = self.renderer_rgb(rgb=field_output_rgb, density=field_output_density, deltas=deltas)
|
|
|
|
print(field_output_rgb)
|
|
print(field_output_density)
|
|
outputs = {
|
|
"rgb_coarse": rgb_coarse
|
|
}
|
|
return outputs
|
|
|
|
def get_losses(self, batch, graph_outputs):
|
|
# batch.pixels # (num_rays, 3)
|
|
losses = {}
|
|
# rgb_loss_coarse = self.rgb_loss(batch.pixels, graph_outputs["rgb_coarse"])
|
|
# rgb_loss_fine = self.rgb_loss(batch.pixels, graph_outputs["rgb_fine"])
|
|
# losses = {"rgb_loss_coarse": rgb_loss_coarse, "rgb_loss_fine": rgb_loss_fine}
|
|
return losses
|