mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 22:25:44 +08:00
Merge branch 'master' into eng/distributed_training
This commit is contained in:
26 files changed
+586
-385
No files matched your search
@@ -0,0 +1,21 @@
|
||||
# dataset options
|
||||
dataset:
|
||||
data_directory: data/lego
|
||||
dataset_type: blender
|
||||
downscale_factor: 1.0
|
||||
|
||||
# dataloader options
|
||||
dataloader:
|
||||
num_workers: 0
|
||||
shuffle: true
|
||||
num_images_to_sample_from: 8
|
||||
num_rays_per_batch: 1024
|
||||
|
||||
# ray generation module and its option
|
||||
network:
|
||||
ray_generator:
|
||||
class_name: RayGenerator
|
||||
inputs: ['cxy'] # camera, x, and y indices
|
||||
meta_data:
|
||||
in_dim: 0
|
||||
learning_rate: 0.0
|
||||
@@ -0,0 +1,75 @@
|
||||
"""
|
||||
Code to test ray generation.
|
||||
"""
|
||||
|
||||
import imageio
|
||||
import torch
|
||||
from hydra import compose, initialize
|
||||
from torch.utils.data import DataLoader
|
||||
from tqdm import tqdm
|
||||
|
||||
from mattport.nerf.dataset.image_dataset import ImageDataset, collate_batch
|
||||
from mattport.nerf.dataset.utils import get_dataset_inputs
|
||||
from mattport.nerf.field_modules.ray_generator import RayGenerator
|
||||
from mattport.utils.io import get_absolute_path
|
||||
from mattport.viewer.plotly import visualize_dataset
|
||||
|
||||
|
||||
def visualize_batch(batch):
|
||||
"""Returns an image from a batch."""
|
||||
assert "image" in batch, "You must call collate_batch with keep_full_image=True for this to work."
|
||||
# set the color of the sampled rays
|
||||
c, y, x = [i.flatten() for i in torch.split(batch.indices, 1, dim=-1)]
|
||||
batch.image[c, y, x] = 1.0
|
||||
|
||||
# batch.image is num_images, h, w, 3
|
||||
images = torch.split(batch.image, 1, dim=0)
|
||||
image_list = [image[0] for image in images]
|
||||
image = torch.cat(image_list, dim=1) # cat along the width dimension
|
||||
image = (image * 255.0).to(torch.uint8)
|
||||
return image
|
||||
|
||||
|
||||
def test_dataloader():
|
||||
"""Testing for the dataloader from input dataset parameters to rays."""
|
||||
|
||||
with initialize(config_path="configs"):
|
||||
cfg = compose(config_name="blender_lego.yaml")
|
||||
|
||||
cfg.dataset.data_directory = get_absolute_path(
|
||||
cfg.dataset.data_directory
|
||||
) # TODO(ethan): create a helper function for this
|
||||
dataset_inputs = get_dataset_inputs(**cfg.dataset)
|
||||
dataset = ImageDataset(
|
||||
image_filenames=dataset_inputs.image_filenames, downscale_factor=dataset_inputs.downscale_factor
|
||||
)
|
||||
|
||||
dataloader = DataLoader(
|
||||
dataset,
|
||||
batch_size=cfg.dataloader.num_images_to_sample_from,
|
||||
collate_fn=lambda batch: collate_batch(batch, cfg.dataloader.num_rays_per_batch, keep_full_image=True),
|
||||
num_workers=cfg.dataloader.num_workers,
|
||||
shuffle=True,
|
||||
)
|
||||
dataloader_iter = iter(dataloader)
|
||||
|
||||
ray_generator = RayGenerator(dataset_inputs.intrinsics, dataset_inputs.camera_to_world)
|
||||
|
||||
num_batches = 10
|
||||
for _ in tqdm(range(num_batches)):
|
||||
batch = next(dataloader_iter)
|
||||
ray_bundle = ray_generator.forward(batch.indices)
|
||||
|
||||
# visualize the batch
|
||||
image = visualize_batch(batch)
|
||||
imageio.imwrite("temp0.png", image)
|
||||
|
||||
# visualize the RayBundle
|
||||
small_ray_bundle = ray_bundle.sample(100)
|
||||
fig = visualize_dataset(camera_origins=dataset_inputs.camera_to_world[:, :3, 3], ray_bundle=small_ray_bundle)
|
||||
fig.write_image("temp1.png")
|
||||
fig.write_html("temp1.html")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_dataloader()
|
||||
@@ -1,56 +0,0 @@
|
||||
"""
|
||||
Test ray generation.
|
||||
"""
|
||||
import torch
|
||||
|
||||
from mattport.nerf.dataset.blender_dataset import load_blender_data
|
||||
from mattport.nerf.field_modules.ray_generator import RayGenerator
|
||||
from mattport.structures.rays import RayBundle
|
||||
from mattport.utils.io import get_absolute_path
|
||||
|
||||
|
||||
def test_ray_generator():
|
||||
"""Test the ray generation works given camera, row, and col indices as input."""
|
||||
|
||||
# load data
|
||||
datadir = get_absolute_path("data/lego")
|
||||
image_filenames, poses, focal_length, image_height, image_width = load_blender_data(
|
||||
datadir, half_res=False, testskip=1
|
||||
)
|
||||
cx = image_width / 2.0
|
||||
cy = image_height / 2.0
|
||||
camera_to_world = torch.from_numpy(poses[:, :3]) # camera to world transform
|
||||
num_cameras = len(image_filenames)
|
||||
num_intrinsics_params = 3
|
||||
intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32)
|
||||
intrinsics *= torch.tensor([focal_length, cx, cy])
|
||||
|
||||
# Initialize the ray generator with the cameras.
|
||||
ray_generator = RayGenerator(intrinsics, camera_to_world)
|
||||
|
||||
# Random ray indices.
|
||||
print("Random ray indexing.")
|
||||
num_rays = 1024
|
||||
ray_indices = torch.floor(torch.rand((num_rays, 3)) * torch.tensor([num_cameras, image_height, image_width])).long()
|
||||
ray_bundle = ray_generator(ray_indices)
|
||||
assert isinstance(ray_bundle, RayBundle)
|
||||
|
||||
# TODO(ethan): visualize the RayBundle object
|
||||
# TODO(ethan): add other test cases? e.g., origin and camera_indices checks
|
||||
|
||||
# check that the origins are correct
|
||||
c = ray_indices[:, 0]
|
||||
assert torch.allclose(ray_bundle.origins, camera_to_world[c, :3, 3])
|
||||
|
||||
# Ray indices for one camera.
|
||||
coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij")
|
||||
coords = torch.stack(coords, dim=-1)
|
||||
coords = coords.view(-1, 2)
|
||||
num_rays = coords.shape[0]
|
||||
camera_index = 0
|
||||
ray_indices = torch.cat([torch.ones_like(coords[:, :1]) * camera_index, coords], dim=-1).long()
|
||||
ray_bundle = ray_generator(ray_indices)
|
||||
|
||||
print(ray_bundle.camera_indices)
|
||||
|
||||
# assert False
|
||||
@@ -0,0 +1,40 @@
|
||||
"""
|
||||
Test renderers
|
||||
"""
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from mattport.nerf import renderers
|
||||
|
||||
|
||||
def test_rgb_renderer():
|
||||
"""Test RGB volumetric rendering"""
|
||||
rgb_name = "rgb"
|
||||
density_name = "density"
|
||||
num_samples = 10
|
||||
|
||||
rgb = torch.ones((3, num_samples, 3))
|
||||
densities = torch.ones((3, num_samples, 1))
|
||||
deltas = torch.ones((3, num_samples))
|
||||
|
||||
rgb_renderer = renderers.RGB(rgb_name, density_name)
|
||||
|
||||
assert rgb_renderer.required_field_outputs is not None
|
||||
|
||||
field_output_dict = {rgb_name: rgb, density_name: densities}
|
||||
out = rgb_renderer(field_output_dict, deltas)
|
||||
assert torch.max(out) > 0.9
|
||||
|
||||
field_output_dict = {rgb_name: rgb * 0, density_name: densities}
|
||||
out = rgb_renderer(field_output_dict, deltas)
|
||||
assert torch.max(out) == pytest.approx(0)
|
||||
|
||||
field_output_dict = {rgb_name: rgb, density_name: densities * 0}
|
||||
out = rgb_renderer(field_output_dict, deltas)
|
||||
assert torch.max(out) == pytest.approx(0)
|
||||
|
||||
# TODO Tancik: Add more precise tests
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_rgb_renderer()
|
||||
@@ -1,12 +1,16 @@
|
||||
"""
|
||||
Test the camera classes.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from mattport.structures.cameras import PinholeCamera
|
||||
from mattport.structures.rays import RayBundle, CameraRayBundle
|
||||
from mattport.structures.rays import CameraRayBundle
|
||||
|
||||
|
||||
def test_pinhole_camera():
|
||||
"""Test that the pinhole camera model works."""
|
||||
pinhole_camera = PinholeCamera(cx=400.0, cy=400.0, fx=10.0, fy=10.0)
|
||||
camera_ray_bundle = pinhole_camera.generate_camera_rays()
|
||||
assert isinstance(camera_ray_bundle, CameraRayBundle)
|
||||
assert torch.allclose(camera_ray_bundle.origins[0], torch.tensor([0.0, 0.0, 0.0]))
|
||||
# assert False
|
||||
Reference in new issue
Block a user