Merge branch 'master' into eng/distributed_training

This commit is contained in:
Ethan Weber authored and GitHub committed 2022-04-17 09:40:30 -07:00
commit e5bc28c4fb
26 files changed
+586 -385

No files matched your search

@@ -0,0 +1,21 @@
# dataset options
dataset:
data_directory: data/lego
dataset_type: blender
downscale_factor: 1.0
# dataloader options
dataloader:
num_workers: 0
shuffle: true
num_images_to_sample_from: 8
num_rays_per_batch: 1024
# ray generation module and its option
network:
ray_generator:
class_name: RayGenerator
inputs: ['cxy'] # camera, x, and y indices
meta_data:
in_dim: 0
learning_rate: 0.0
@@ -0,0 +1,75 @@
"""
Code to test ray generation.
"""
import imageio
import torch
from hydra import compose, initialize
from torch.utils.data import DataLoader
from tqdm import tqdm
from mattport.nerf.dataset.image_dataset import ImageDataset, collate_batch
from mattport.nerf.dataset.utils import get_dataset_inputs
from mattport.nerf.field_modules.ray_generator import RayGenerator
from mattport.utils.io import get_absolute_path
from mattport.viewer.plotly import visualize_dataset
def visualize_batch(batch):
"""Returns an image from a batch."""
assert "image" in batch, "You must call collate_batch with keep_full_image=True for this to work."
# set the color of the sampled rays
c, y, x = [i.flatten() for i in torch.split(batch.indices, 1, dim=-1)]
batch.image[c, y, x] = 1.0
# batch.image is num_images, h, w, 3
images = torch.split(batch.image, 1, dim=0)
image_list = [image[0] for image in images]
image = torch.cat(image_list, dim=1) # cat along the width dimension
image = (image * 255.0).to(torch.uint8)
return image
def test_dataloader():
"""Testing for the dataloader from input dataset parameters to rays."""
with initialize(config_path="configs"):
cfg = compose(config_name="blender_lego.yaml")
cfg.dataset.data_directory = get_absolute_path(
cfg.dataset.data_directory
) # TODO(ethan): create a helper function for this
dataset_inputs = get_dataset_inputs(**cfg.dataset)
dataset = ImageDataset(
image_filenames=dataset_inputs.image_filenames, downscale_factor=dataset_inputs.downscale_factor
)
dataloader = DataLoader(
dataset,
batch_size=cfg.dataloader.num_images_to_sample_from,
collate_fn=lambda batch: collate_batch(batch, cfg.dataloader.num_rays_per_batch, keep_full_image=True),
num_workers=cfg.dataloader.num_workers,
shuffle=True,
)
dataloader_iter = iter(dataloader)
ray_generator = RayGenerator(dataset_inputs.intrinsics, dataset_inputs.camera_to_world)
num_batches = 10
for _ in tqdm(range(num_batches)):
batch = next(dataloader_iter)
ray_bundle = ray_generator.forward(batch.indices)
# visualize the batch
image = visualize_batch(batch)
imageio.imwrite("temp0.png", image)
# visualize the RayBundle
small_ray_bundle = ray_bundle.sample(100)
fig = visualize_dataset(camera_origins=dataset_inputs.camera_to_world[:, :3, 3], ray_bundle=small_ray_bundle)
fig.write_image("temp1.png")
fig.write_html("temp1.html")
if __name__ == "__main__":
test_dataloader()
-56
View File
@@ -1,56 +0,0 @@
"""
Test ray generation.
"""
import torch
from mattport.nerf.dataset.blender_dataset import load_blender_data
from mattport.nerf.field_modules.ray_generator import RayGenerator
from mattport.structures.rays import RayBundle
from mattport.utils.io import get_absolute_path
def test_ray_generator():
"""Test the ray generation works given camera, row, and col indices as input."""
# load data
datadir = get_absolute_path("data/lego")
image_filenames, poses, focal_length, image_height, image_width = load_blender_data(
datadir, half_res=False, testskip=1
)
cx = image_width / 2.0
cy = image_height / 2.0
camera_to_world = torch.from_numpy(poses[:, :3]) # camera to world transform
num_cameras = len(image_filenames)
num_intrinsics_params = 3
intrinsics = torch.ones((num_cameras, num_intrinsics_params), dtype=torch.float32)
intrinsics *= torch.tensor([focal_length, cx, cy])
# Initialize the ray generator with the cameras.
ray_generator = RayGenerator(intrinsics, camera_to_world)
# Random ray indices.
print("Random ray indexing.")
num_rays = 1024
ray_indices = torch.floor(torch.rand((num_rays, 3)) * torch.tensor([num_cameras, image_height, image_width])).long()
ray_bundle = ray_generator(ray_indices)
assert isinstance(ray_bundle, RayBundle)
# TODO(ethan): visualize the RayBundle object
# TODO(ethan): add other test cases? e.g., origin and camera_indices checks
# check that the origins are correct
c = ray_indices[:, 0]
assert torch.allclose(ray_bundle.origins, camera_to_world[c, :3, 3])
# Ray indices for one camera.
coords = torch.meshgrid(torch.arange(image_height), torch.arange(image_width), indexing="ij")
coords = torch.stack(coords, dim=-1)
coords = coords.view(-1, 2)
num_rays = coords.shape[0]
camera_index = 0
ray_indices = torch.cat([torch.ones_like(coords[:, :1]) * camera_index, coords], dim=-1).long()
ray_bundle = ray_generator(ray_indices)
print(ray_bundle.camera_indices)
# assert False
+40
View File
@@ -0,0 +1,40 @@
"""
Test renderers
"""
import pytest
import torch
from mattport.nerf import renderers
def test_rgb_renderer():
"""Test RGB volumetric rendering"""
rgb_name = "rgb"
density_name = "density"
num_samples = 10
rgb = torch.ones((3, num_samples, 3))
densities = torch.ones((3, num_samples, 1))
deltas = torch.ones((3, num_samples))
rgb_renderer = renderers.RGB(rgb_name, density_name)
assert rgb_renderer.required_field_outputs is not None
field_output_dict = {rgb_name: rgb, density_name: densities}
out = rgb_renderer(field_output_dict, deltas)
assert torch.max(out) > 0.9
field_output_dict = {rgb_name: rgb * 0, density_name: densities}
out = rgb_renderer(field_output_dict, deltas)
assert torch.max(out) == pytest.approx(0)
field_output_dict = {rgb_name: rgb, density_name: densities * 0}
out = rgb_renderer(field_output_dict, deltas)
assert torch.max(out) == pytest.approx(0)
# TODO Tancik: Add more precise tests
if __name__ == "__main__":
test_rgb_renderer()
+6 -2
View File
@@ -1,12 +1,16 @@
"""
Test the camera classes.
"""
import torch
from mattport.structures.cameras import PinholeCamera
from mattport.structures.rays import RayBundle, CameraRayBundle
from mattport.structures.rays import CameraRayBundle
def test_pinhole_camera():
"""Test that the pinhole camera model works."""
pinhole_camera = PinholeCamera(cx=400.0, cy=400.0, fx=10.0, fy=10.0)
camera_ray_bundle = pinhole_camera.generate_camera_rays()
assert isinstance(camera_ray_bundle, CameraRayBundle)
assert torch.allclose(camera_ray_bundle.origins[0], torch.tensor([0.0, 0.0, 0.0]))
# assert False