merge main

This commit is contained in:
Justin Kerr committed 2023-09-22 16:29:03 -07:00
commit 3c40314b62
52 files changed
+2035 -299

No files matched your search

+142 -25
View File
@@ -19,6 +19,7 @@ import math
import sys
from dataclasses import dataclass, field
from pathlib import Path
from functools import partial
from typing import List, Literal, Optional, Type
import numpy as np
@@ -32,8 +33,8 @@ from nerfstudio.data.dataparsers.base_dataparser import DataParser, DataParserCo
from nerfstudio.data.scene_box import SceneBox
from nerfstudio.data.utils import colmap_parsing_utils as colmap_utils
from nerfstudio.process_data.colmap_utils import parse_colmap_camera_params
from nerfstudio.process_data.process_data_utils import downscale_images
from nerfstudio.utils.rich_utils import CONSOLE
from nerfstudio.utils.scripts import run_command
from nerfstudio.utils.rich_utils import CONSOLE, status
MAX_AUTO_RESOLUTION = 1600
@@ -71,6 +72,10 @@ class ColmapDataParserConfig(DataParserConfig):
"""Path to depth maps directory. If not set, depths are not loaded."""
colmap_path: Path = Path("sparse/0")
"""Path to the colmap reconstruction directory relative to the data path."""
load_3D_points: bool = False
"""Whether to load the 3D points from the colmap reconstruction."""
max_2D_matches_per_3D_point: int = -1
"""Maximum number of 2D matches per 3D point. If set to -1, all 2D matches are loaded. If set to 0, no 2D matches are loaded."""
class ColmapDataParser(DataParser):
@@ -135,15 +140,19 @@ class ColmapDataParser(DataParser):
c2w[2, :] *= -1
frame = {
"file_path": (self.config.images_path / im_data.name).as_posix(),
"file_path": (self.config.data / self.config.images_path / im_data.name).as_posix(),
"transform_matrix": c2w,
"colmap_im_id": im_id,
}
frame.update(cameras[im_data.camera_id])
if self.config.masks_path is not None:
frame["mask_path"] = ((self.config.masks_path / im_data.name).with_suffix(".png").as_posix(),)
frame["mask_path"] = (
(self.config.data / self.config.masks_path / im_data.name).with_suffix(".png").as_posix(),
)
if self.config.depths_path is not None:
frame["depth_path"] = ((self.config.depths_path / im_data.name).with_suffix(".png").as_posix(),)
frame["depth_path"] = (
(self.config.data / self.config.depths_path / im_data.name).with_suffix(".png").as_posix(),
)
frames.append(frame)
if camera_model is not None:
assert camera_model == frame["camera_model"], "Multiple camera models are not supported"
@@ -171,7 +180,7 @@ class ColmapDataParser(DataParser):
with (self.config.data / f"{split}_list.txt").open("r", encoding="utf8") as f:
filenames = f.read().splitlines()
# Validate split first
split_filenames = set(self.config.images_path / x for x in filenames)
split_filenames = set(self.config.data / self.config.images_path / x for x in filenames)
unmatched_filenames = split_filenames.difference(image_filenames)
if unmatched_filenames:
raise RuntimeError(
@@ -202,7 +211,7 @@ class ColmapDataParser(DataParser):
raise ValueError(f"Unknown dataparser split {split}")
return indices
def _generate_dataparser_outputs(self, split: str = "train"):
def _generate_dataparser_outputs(self, split: str = "train", **kwargs):
assert self.config.data.exists(), f"Data directory {self.config.data} does not exist."
colmap_path = self.config.data / self.config.colmap_path
assert colmap_path.exists(), f"Colmap path {colmap_path} does not exist."
@@ -328,6 +337,11 @@ class ColmapDataParser(DataParser):
applied_scale = float(meta["applied_scale"])
scale_factor *= applied_scale
metadata = {}
if self.config.load_3D_points:
# Load 3D points
metadata.update(self._load_3D_points(colmap_path, transform_matrix, scale_factor))
dataparser_outputs = DataparserOutputs(
image_filenames=image_filenames,
cameras=cameras,
@@ -338,10 +352,96 @@ class ColmapDataParser(DataParser):
metadata={
"depth_filenames": depth_filenames if len(depth_filenames) > 0 else None,
"depth_unit_scale_factor": self.config.depth_unit_scale_factor,
**metadata,
},
)
return dataparser_outputs
def _load_3D_points(self, colmap_path: Path, transform_matrix: torch.Tensor, scale_factor: float):
if (colmap_path / "points3D.bin").exists():
colmap_points = colmap_utils.read_points3D_binary(colmap_path / "points3D.bin")
elif (colmap_path / "points3D.txt").exists():
colmap_points = colmap_utils.read_points3D_text(colmap_path / "points3D.txt")
else:
raise ValueError(f"Could not find points3D.txt or points3D.bin in {colmap_path}")
points3D = torch.from_numpy(np.array([p.xyz for p in colmap_points.values()], dtype=np.float32))
points3D = (
torch.cat(
(
points3D,
torch.ones_like(points3D[..., :1]),
),
-1,
)
@ transform_matrix.T
)
points3D *= scale_factor
# Load point colours
points3D_rgb = torch.from_numpy(np.array([p.rgb for p in colmap_points.values()], dtype=np.uint8))
points3D_num_points = torch.tensor([len(p.image_ids) for p in colmap_points.values()], dtype=torch.int64)
out = {
"points3D_xyz": points3D,
"points3D_rgb": points3D_rgb,
"points3D_error": torch.from_numpy(np.array([p.error for p in colmap_points.values()], dtype=np.float32)),
"points3D_num_points2D": points3D_num_points,
}
if self.config.max_2D_matches_per_3D_point != 0:
if (colmap_path / "images.txt").exists():
im_id_to_image = colmap_utils.read_images_text(colmap_path / "images.txt")
elif (colmap_path / "images.bin").exists():
im_id_to_image = colmap_utils.read_images_binary(colmap_path / "images.bin")
else:
raise ValueError(f"Could not find images.txt or images.bin in {colmap_path}")
downscale_factor = self._downscale_factor
max_num_points = int(torch.max(points3D_num_points).item())
if self.config.max_2D_matches_per_3D_point > 0:
max_num_points = min(max_num_points, self.config.max_2D_matches_per_3D_point)
points3D_image_ids = []
points3D_image_xy = []
for p in colmap_points.values():
nids = np.array(p.image_ids, dtype=np.int64)
nxy_ids = np.array(p.point2D_idxs, dtype=np.int32)
if self.config.max_2D_matches_per_3D_point != -1:
# Randomly sample 2D matches
idxs = np.argsort(p.error)[: self.config.max_2D_matches_per_3D_point]
nids = nids[idxs]
nxy_ids = nxy_ids[idxs]
nxy = [im_id_to_image[im_id].xys[pt_idx] for im_id, pt_idx in zip(nids, nxy_ids)]
nxy = torch.from_numpy(np.stack(nxy).astype(np.float32))
nids = torch.from_numpy(nids)
assert len(nids.shape) == 1
assert len(nxy.shape) == 2
points3D_image_ids.append(
torch.cat((nids, torch.full((max_num_points - len(nids),), -1, dtype=torch.int64)))
)
points3D_image_xy.append(
torch.cat((nxy, torch.full((max_num_points - len(nxy), nxy.shape[-1]), 0, dtype=torch.float32)))
/ downscale_factor
)
out["points3D_image_ids"] = torch.stack(points3D_image_ids, dim=0)
out["points3D_points2D_xy"] = torch.stack(points3D_image_xy, dim=0)
return out
def _downscale_images(self, paths, get_fname, downscale_factor: int, nearest_neighbor: bool = False):
with status(msg="[bold yellow]Downscaling images...", spinner="growVertical"):
assert downscale_factor > 1
assert isinstance(downscale_factor, int)
# Using %05d ffmpeg commands appears to be unreliable (skips images).
for path in paths:
nn_flag = "" if not nearest_neighbor else ":flags=neighbor"
path_out = get_fname(path)
path_out.parent.mkdir(parents=True, exist_ok=True)
ffmpeg_cmd = [
f'ffmpeg -y -noautorotate -i "{path}" ',
f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor}{nn_flag} ",
f'"{path_out}"',
]
ffmpeg_cmd = " ".join(ffmpeg_cmd)
run_command(ffmpeg_cmd)
CONSOLE.log("[bold green]:tada: Done downscaling images.")
def _setup_downscale_factor(
self, image_filenames: List[Path], mask_filenames: List[Path], depth_filenames: List[Path]
):
@@ -349,17 +449,16 @@ class ColmapDataParser(DataParser):
Setup the downscale factor for the dataset. This is used to downscale the images and cameras.
"""
def get_fname(filepath: Path) -> Path:
def get_fname(parent: Path, filepath: Path) -> Path:
"""Returns transformed file name when downscale factor is applied"""
parts = list(filepath.parts)
parts[-2] += f"_{self._downscale_factor}"
filepath = Path(*parts)
return self.config.data / filepath
rel_part = filepath.relative_to(parent)
base_part = parent.parent / (str(parent.name) + f"_{self._downscale_factor}")
return base_part / rel_part
filepath = next(iter(image_filenames))
if self._downscale_factor is None:
if self.config.downscale_factor is None:
test_img = Image.open(self.config.data / filepath)
test_img = Image.open(filepath)
h, w = test_img.size
max_res = max(h, w)
df = 0
@@ -372,7 +471,9 @@ class ColmapDataParser(DataParser):
CONSOLE.log(f"Using image downscale factor of {self._downscale_factor}")
else:
self._downscale_factor = self.config.downscale_factor
if self._downscale_factor > 1 and not all(get_fname(fp).parent.exists() for fp in image_filenames):
if self._downscale_factor > 1 and not all(
get_fname(self.config.data / self.config.images_path, fp).parent.exists() for fp in image_filenames
):
# Downscaled images not found
# Ask if user wants to downscale the images automatically here
CONSOLE.print(
@@ -380,23 +481,39 @@ class ColmapDataParser(DataParser):
)
if Confirm.ask("\nWould you like to downscale the images now?", default=False, console=CONSOLE):
# Install the method
image_dir = self.config.data / image_filenames[0].parent
num_downscales = int(math.log2(self._downscale_factor))
assert 2**num_downscales == self._downscale_factor, "Downscale factor must be a power of 2"
downscale_images(image_dir, num_downscales, folder_name=image_dir.name, nearest_neighbor=False)
self._downscale_images(
image_filenames,
partial(get_fname, self.config.data / self.config.images_path),
self._downscale_factor,
nearest_neighbor=False,
)
if len(mask_filenames) > 0:
mask_dir = mask_filenames[0].parent
downscale_images(mask_dir, num_downscales, folder_name=mask_dir.name, nearest_neighbor=True)
assert self.config.masks_path is not None
self._downscale_images(
mask_filenames,
partial(get_fname, self.config.data / self.config.masks_path),
self._downscale_factor,
nearest_neighbor=True,
)
if len(depth_filenames) > 0:
depth_dir = depth_filenames[0].parent
downscale_images(depth_dir, num_downscales, folder_name=depth_dir.name, nearest_neighbor=False)
assert self.config.depths_path is not None
self._downscale_images(
depth_filenames,
partial(get_fname, self.config.data / self.config.depths_path),
self._downscale_factor,
nearest_neighbor=True,
)
else:
sys.exit(1)
# Return transformed filenames
if self._downscale_factor > 1:
image_filenames = [get_fname(fp) for fp in image_filenames]
mask_filenames = [get_fname(fp) for fp in mask_filenames]
depth_filenames = [get_fname(fp) for fp in depth_filenames]
image_filenames = [get_fname(self.config.data / self.config.images_path, fp) for fp in image_filenames]
if len(mask_filenames) > 0:
assert self.config.masks_path is not None
mask_filenames = [get_fname(self.config.data / self.config.masks_path, fp) for fp in mask_filenames]
if len(depth_filenames) > 0:
assert self.config.depths_path is not None
depth_filenames = [get_fname(self.config.data / self.config.depths_path, fp) for fp in depth_filenames]
assert isinstance(self._downscale_factor, int)
return image_filenames, mask_filenames, depth_filenames, self._downscale_factor
+1 -1
View File
@@ -215,7 +215,7 @@ class PixelSampler:
num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch
indices = self.sample_method(
num_rays_in_batch, 1, image_height, image_width, mask=batch["mask"][i], device=device
num_rays_in_batch, 1, image_height, image_width, mask=batch["mask"][i].unsqueeze(0), device=device
)
indices[:, 0] = i
all_indices.append(indices)
+37 -1
View File
@@ -17,7 +17,8 @@ Dataset input structures.
"""
from dataclasses import dataclass
from typing import Union
from typing import Union, Tuple
import viser.transforms as vtf
import torch
from jaxtyping import Float
@@ -76,3 +77,38 @@ class SceneBox:
xyzs = poses[..., :3, -1]
aabb = torch.stack([torch.min(xyzs, dim=0)[0], torch.max(xyzs, dim=0)[0]])
return SceneBox(aabb=aabb * scale_factor)
@dataclass
class OrientedBox:
R: Float[Tensor, "3 3"]
"""R: rotation matrix."""
T: Float[Tensor, "3"]
"""T: translation vector."""
S: Float[Tensor, "3"]
"""S: scale vector."""
def within(self, pts: Float[Tensor, "n 3"]):
"""Returns a boolean mask indicating whether each point is within the box."""
R, T, S = self.R, self.T, self.S.to(pts)
H = torch.eye(4, device=pts.device, dtype=pts.dtype)
H[:3, :3] = R
H[:3, 3] = T
H_world2bbox = torch.inverse(H)
pts = torch.cat((pts, torch.ones_like(pts[..., :1])), dim=-1)
pts = torch.matmul(H_world2bbox, pts.T).T[..., :3]
comp_l = torch.tensor(-S / 2)
comp_m = torch.tensor(S / 2)
mask = torch.all(torch.concat([pts > comp_l, pts < comp_m], dim=-1), dim=-1)
return mask
@staticmethod
def from_params(
pos: Tuple[float, float, float], rpy: Tuple[float, float, float], scale: Tuple[float, float, float]
):
"""Construct a box from position, rotation, and scale parameters."""
R = torch.tensor(vtf.SO3.from_rpy_radians(rpy[0], rpy[1], rpy[2]).as_matrix())
T = torch.tensor(pos)
S = torch.tensor(scale)
return OrientedBox(R=R, T=T, S=S)