mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 15:15:43 +08:00
merge main
This commit is contained in:
commit
3c40314b62
52 files changed
+2035
-299
No files matched your search
@@ -19,6 +19,7 @@ import math
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from functools import partial
|
||||
from typing import List, Literal, Optional, Type
|
||||
|
||||
import numpy as np
|
||||
@@ -32,8 +33,8 @@ from nerfstudio.data.dataparsers.base_dataparser import DataParser, DataParserCo
|
||||
from nerfstudio.data.scene_box import SceneBox
|
||||
from nerfstudio.data.utils import colmap_parsing_utils as colmap_utils
|
||||
from nerfstudio.process_data.colmap_utils import parse_colmap_camera_params
|
||||
from nerfstudio.process_data.process_data_utils import downscale_images
|
||||
from nerfstudio.utils.rich_utils import CONSOLE
|
||||
from nerfstudio.utils.scripts import run_command
|
||||
from nerfstudio.utils.rich_utils import CONSOLE, status
|
||||
|
||||
MAX_AUTO_RESOLUTION = 1600
|
||||
|
||||
@@ -71,6 +72,10 @@ class ColmapDataParserConfig(DataParserConfig):
|
||||
"""Path to depth maps directory. If not set, depths are not loaded."""
|
||||
colmap_path: Path = Path("sparse/0")
|
||||
"""Path to the colmap reconstruction directory relative to the data path."""
|
||||
load_3D_points: bool = False
|
||||
"""Whether to load the 3D points from the colmap reconstruction."""
|
||||
max_2D_matches_per_3D_point: int = -1
|
||||
"""Maximum number of 2D matches per 3D point. If set to -1, all 2D matches are loaded. If set to 0, no 2D matches are loaded."""
|
||||
|
||||
|
||||
class ColmapDataParser(DataParser):
|
||||
@@ -135,15 +140,19 @@ class ColmapDataParser(DataParser):
|
||||
c2w[2, :] *= -1
|
||||
|
||||
frame = {
|
||||
"file_path": (self.config.images_path / im_data.name).as_posix(),
|
||||
"file_path": (self.config.data / self.config.images_path / im_data.name).as_posix(),
|
||||
"transform_matrix": c2w,
|
||||
"colmap_im_id": im_id,
|
||||
}
|
||||
frame.update(cameras[im_data.camera_id])
|
||||
if self.config.masks_path is not None:
|
||||
frame["mask_path"] = ((self.config.masks_path / im_data.name).with_suffix(".png").as_posix(),)
|
||||
frame["mask_path"] = (
|
||||
(self.config.data / self.config.masks_path / im_data.name).with_suffix(".png").as_posix(),
|
||||
)
|
||||
if self.config.depths_path is not None:
|
||||
frame["depth_path"] = ((self.config.depths_path / im_data.name).with_suffix(".png").as_posix(),)
|
||||
frame["depth_path"] = (
|
||||
(self.config.data / self.config.depths_path / im_data.name).with_suffix(".png").as_posix(),
|
||||
)
|
||||
frames.append(frame)
|
||||
if camera_model is not None:
|
||||
assert camera_model == frame["camera_model"], "Multiple camera models are not supported"
|
||||
@@ -171,7 +180,7 @@ class ColmapDataParser(DataParser):
|
||||
with (self.config.data / f"{split}_list.txt").open("r", encoding="utf8") as f:
|
||||
filenames = f.read().splitlines()
|
||||
# Validate split first
|
||||
split_filenames = set(self.config.images_path / x for x in filenames)
|
||||
split_filenames = set(self.config.data / self.config.images_path / x for x in filenames)
|
||||
unmatched_filenames = split_filenames.difference(image_filenames)
|
||||
if unmatched_filenames:
|
||||
raise RuntimeError(
|
||||
@@ -202,7 +211,7 @@ class ColmapDataParser(DataParser):
|
||||
raise ValueError(f"Unknown dataparser split {split}")
|
||||
return indices
|
||||
|
||||
def _generate_dataparser_outputs(self, split: str = "train"):
|
||||
def _generate_dataparser_outputs(self, split: str = "train", **kwargs):
|
||||
assert self.config.data.exists(), f"Data directory {self.config.data} does not exist."
|
||||
colmap_path = self.config.data / self.config.colmap_path
|
||||
assert colmap_path.exists(), f"Colmap path {colmap_path} does not exist."
|
||||
@@ -328,6 +337,11 @@ class ColmapDataParser(DataParser):
|
||||
applied_scale = float(meta["applied_scale"])
|
||||
scale_factor *= applied_scale
|
||||
|
||||
metadata = {}
|
||||
if self.config.load_3D_points:
|
||||
# Load 3D points
|
||||
metadata.update(self._load_3D_points(colmap_path, transform_matrix, scale_factor))
|
||||
|
||||
dataparser_outputs = DataparserOutputs(
|
||||
image_filenames=image_filenames,
|
||||
cameras=cameras,
|
||||
@@ -338,10 +352,96 @@ class ColmapDataParser(DataParser):
|
||||
metadata={
|
||||
"depth_filenames": depth_filenames if len(depth_filenames) > 0 else None,
|
||||
"depth_unit_scale_factor": self.config.depth_unit_scale_factor,
|
||||
**metadata,
|
||||
},
|
||||
)
|
||||
return dataparser_outputs
|
||||
|
||||
def _load_3D_points(self, colmap_path: Path, transform_matrix: torch.Tensor, scale_factor: float):
|
||||
if (colmap_path / "points3D.bin").exists():
|
||||
colmap_points = colmap_utils.read_points3D_binary(colmap_path / "points3D.bin")
|
||||
elif (colmap_path / "points3D.txt").exists():
|
||||
colmap_points = colmap_utils.read_points3D_text(colmap_path / "points3D.txt")
|
||||
else:
|
||||
raise ValueError(f"Could not find points3D.txt or points3D.bin in {colmap_path}")
|
||||
points3D = torch.from_numpy(np.array([p.xyz for p in colmap_points.values()], dtype=np.float32))
|
||||
points3D = (
|
||||
torch.cat(
|
||||
(
|
||||
points3D,
|
||||
torch.ones_like(points3D[..., :1]),
|
||||
),
|
||||
-1,
|
||||
)
|
||||
@ transform_matrix.T
|
||||
)
|
||||
points3D *= scale_factor
|
||||
|
||||
# Load point colours
|
||||
points3D_rgb = torch.from_numpy(np.array([p.rgb for p in colmap_points.values()], dtype=np.uint8))
|
||||
points3D_num_points = torch.tensor([len(p.image_ids) for p in colmap_points.values()], dtype=torch.int64)
|
||||
out = {
|
||||
"points3D_xyz": points3D,
|
||||
"points3D_rgb": points3D_rgb,
|
||||
"points3D_error": torch.from_numpy(np.array([p.error for p in colmap_points.values()], dtype=np.float32)),
|
||||
"points3D_num_points2D": points3D_num_points,
|
||||
}
|
||||
if self.config.max_2D_matches_per_3D_point != 0:
|
||||
if (colmap_path / "images.txt").exists():
|
||||
im_id_to_image = colmap_utils.read_images_text(colmap_path / "images.txt")
|
||||
elif (colmap_path / "images.bin").exists():
|
||||
im_id_to_image = colmap_utils.read_images_binary(colmap_path / "images.bin")
|
||||
else:
|
||||
raise ValueError(f"Could not find images.txt or images.bin in {colmap_path}")
|
||||
downscale_factor = self._downscale_factor
|
||||
max_num_points = int(torch.max(points3D_num_points).item())
|
||||
if self.config.max_2D_matches_per_3D_point > 0:
|
||||
max_num_points = min(max_num_points, self.config.max_2D_matches_per_3D_point)
|
||||
points3D_image_ids = []
|
||||
points3D_image_xy = []
|
||||
for p in colmap_points.values():
|
||||
nids = np.array(p.image_ids, dtype=np.int64)
|
||||
nxy_ids = np.array(p.point2D_idxs, dtype=np.int32)
|
||||
if self.config.max_2D_matches_per_3D_point != -1:
|
||||
# Randomly sample 2D matches
|
||||
idxs = np.argsort(p.error)[: self.config.max_2D_matches_per_3D_point]
|
||||
nids = nids[idxs]
|
||||
nxy_ids = nxy_ids[idxs]
|
||||
nxy = [im_id_to_image[im_id].xys[pt_idx] for im_id, pt_idx in zip(nids, nxy_ids)]
|
||||
nxy = torch.from_numpy(np.stack(nxy).astype(np.float32))
|
||||
nids = torch.from_numpy(nids)
|
||||
assert len(nids.shape) == 1
|
||||
assert len(nxy.shape) == 2
|
||||
points3D_image_ids.append(
|
||||
torch.cat((nids, torch.full((max_num_points - len(nids),), -1, dtype=torch.int64)))
|
||||
)
|
||||
points3D_image_xy.append(
|
||||
torch.cat((nxy, torch.full((max_num_points - len(nxy), nxy.shape[-1]), 0, dtype=torch.float32)))
|
||||
/ downscale_factor
|
||||
)
|
||||
out["points3D_image_ids"] = torch.stack(points3D_image_ids, dim=0)
|
||||
out["points3D_points2D_xy"] = torch.stack(points3D_image_xy, dim=0)
|
||||
return out
|
||||
|
||||
def _downscale_images(self, paths, get_fname, downscale_factor: int, nearest_neighbor: bool = False):
|
||||
with status(msg="[bold yellow]Downscaling images...", spinner="growVertical"):
|
||||
assert downscale_factor > 1
|
||||
assert isinstance(downscale_factor, int)
|
||||
# Using %05d ffmpeg commands appears to be unreliable (skips images).
|
||||
for path in paths:
|
||||
nn_flag = "" if not nearest_neighbor else ":flags=neighbor"
|
||||
path_out = get_fname(path)
|
||||
path_out.parent.mkdir(parents=True, exist_ok=True)
|
||||
ffmpeg_cmd = [
|
||||
f'ffmpeg -y -noautorotate -i "{path}" ',
|
||||
f"-q:v 2 -vf scale=iw/{downscale_factor}:ih/{downscale_factor}{nn_flag} ",
|
||||
f'"{path_out}"',
|
||||
]
|
||||
ffmpeg_cmd = " ".join(ffmpeg_cmd)
|
||||
run_command(ffmpeg_cmd)
|
||||
|
||||
CONSOLE.log("[bold green]:tada: Done downscaling images.")
|
||||
|
||||
def _setup_downscale_factor(
|
||||
self, image_filenames: List[Path], mask_filenames: List[Path], depth_filenames: List[Path]
|
||||
):
|
||||
@@ -349,17 +449,16 @@ class ColmapDataParser(DataParser):
|
||||
Setup the downscale factor for the dataset. This is used to downscale the images and cameras.
|
||||
"""
|
||||
|
||||
def get_fname(filepath: Path) -> Path:
|
||||
def get_fname(parent: Path, filepath: Path) -> Path:
|
||||
"""Returns transformed file name when downscale factor is applied"""
|
||||
parts = list(filepath.parts)
|
||||
parts[-2] += f"_{self._downscale_factor}"
|
||||
filepath = Path(*parts)
|
||||
return self.config.data / filepath
|
||||
rel_part = filepath.relative_to(parent)
|
||||
base_part = parent.parent / (str(parent.name) + f"_{self._downscale_factor}")
|
||||
return base_part / rel_part
|
||||
|
||||
filepath = next(iter(image_filenames))
|
||||
if self._downscale_factor is None:
|
||||
if self.config.downscale_factor is None:
|
||||
test_img = Image.open(self.config.data / filepath)
|
||||
test_img = Image.open(filepath)
|
||||
h, w = test_img.size
|
||||
max_res = max(h, w)
|
||||
df = 0
|
||||
@@ -372,7 +471,9 @@ class ColmapDataParser(DataParser):
|
||||
CONSOLE.log(f"Using image downscale factor of {self._downscale_factor}")
|
||||
else:
|
||||
self._downscale_factor = self.config.downscale_factor
|
||||
if self._downscale_factor > 1 and not all(get_fname(fp).parent.exists() for fp in image_filenames):
|
||||
if self._downscale_factor > 1 and not all(
|
||||
get_fname(self.config.data / self.config.images_path, fp).parent.exists() for fp in image_filenames
|
||||
):
|
||||
# Downscaled images not found
|
||||
# Ask if user wants to downscale the images automatically here
|
||||
CONSOLE.print(
|
||||
@@ -380,23 +481,39 @@ class ColmapDataParser(DataParser):
|
||||
)
|
||||
if Confirm.ask("\nWould you like to downscale the images now?", default=False, console=CONSOLE):
|
||||
# Install the method
|
||||
image_dir = self.config.data / image_filenames[0].parent
|
||||
num_downscales = int(math.log2(self._downscale_factor))
|
||||
assert 2**num_downscales == self._downscale_factor, "Downscale factor must be a power of 2"
|
||||
downscale_images(image_dir, num_downscales, folder_name=image_dir.name, nearest_neighbor=False)
|
||||
self._downscale_images(
|
||||
image_filenames,
|
||||
partial(get_fname, self.config.data / self.config.images_path),
|
||||
self._downscale_factor,
|
||||
nearest_neighbor=False,
|
||||
)
|
||||
if len(mask_filenames) > 0:
|
||||
mask_dir = mask_filenames[0].parent
|
||||
downscale_images(mask_dir, num_downscales, folder_name=mask_dir.name, nearest_neighbor=True)
|
||||
assert self.config.masks_path is not None
|
||||
self._downscale_images(
|
||||
mask_filenames,
|
||||
partial(get_fname, self.config.data / self.config.masks_path),
|
||||
self._downscale_factor,
|
||||
nearest_neighbor=True,
|
||||
)
|
||||
if len(depth_filenames) > 0:
|
||||
depth_dir = depth_filenames[0].parent
|
||||
downscale_images(depth_dir, num_downscales, folder_name=depth_dir.name, nearest_neighbor=False)
|
||||
assert self.config.depths_path is not None
|
||||
self._downscale_images(
|
||||
depth_filenames,
|
||||
partial(get_fname, self.config.data / self.config.depths_path),
|
||||
self._downscale_factor,
|
||||
nearest_neighbor=True,
|
||||
)
|
||||
else:
|
||||
sys.exit(1)
|
||||
|
||||
# Return transformed filenames
|
||||
if self._downscale_factor > 1:
|
||||
image_filenames = [get_fname(fp) for fp in image_filenames]
|
||||
mask_filenames = [get_fname(fp) for fp in mask_filenames]
|
||||
depth_filenames = [get_fname(fp) for fp in depth_filenames]
|
||||
image_filenames = [get_fname(self.config.data / self.config.images_path, fp) for fp in image_filenames]
|
||||
if len(mask_filenames) > 0:
|
||||
assert self.config.masks_path is not None
|
||||
mask_filenames = [get_fname(self.config.data / self.config.masks_path, fp) for fp in mask_filenames]
|
||||
if len(depth_filenames) > 0:
|
||||
assert self.config.depths_path is not None
|
||||
depth_filenames = [get_fname(self.config.data / self.config.depths_path, fp) for fp in depth_filenames]
|
||||
assert isinstance(self._downscale_factor, int)
|
||||
return image_filenames, mask_filenames, depth_filenames, self._downscale_factor
|
||||
@@ -215,7 +215,7 @@ class PixelSampler:
|
||||
num_rays_in_batch = num_rays_per_batch - (num_images - 1) * num_rays_in_batch
|
||||
|
||||
indices = self.sample_method(
|
||||
num_rays_in_batch, 1, image_height, image_width, mask=batch["mask"][i], device=device
|
||||
num_rays_in_batch, 1, image_height, image_width, mask=batch["mask"][i].unsqueeze(0), device=device
|
||||
)
|
||||
indices[:, 0] = i
|
||||
all_indices.append(indices)
|
||||
|
||||
@@ -17,7 +17,8 @@ Dataset input structures.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Union
|
||||
from typing import Union, Tuple
|
||||
import viser.transforms as vtf
|
||||
|
||||
import torch
|
||||
from jaxtyping import Float
|
||||
@@ -76,3 +77,38 @@ class SceneBox:
|
||||
xyzs = poses[..., :3, -1]
|
||||
aabb = torch.stack([torch.min(xyzs, dim=0)[0], torch.max(xyzs, dim=0)[0]])
|
||||
return SceneBox(aabb=aabb * scale_factor)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OrientedBox:
|
||||
R: Float[Tensor, "3 3"]
|
||||
"""R: rotation matrix."""
|
||||
T: Float[Tensor, "3"]
|
||||
"""T: translation vector."""
|
||||
S: Float[Tensor, "3"]
|
||||
"""S: scale vector."""
|
||||
|
||||
def within(self, pts: Float[Tensor, "n 3"]):
|
||||
"""Returns a boolean mask indicating whether each point is within the box."""
|
||||
R, T, S = self.R, self.T, self.S.to(pts)
|
||||
H = torch.eye(4, device=pts.device, dtype=pts.dtype)
|
||||
H[:3, :3] = R
|
||||
H[:3, 3] = T
|
||||
H_world2bbox = torch.inverse(H)
|
||||
pts = torch.cat((pts, torch.ones_like(pts[..., :1])), dim=-1)
|
||||
pts = torch.matmul(H_world2bbox, pts.T).T[..., :3]
|
||||
|
||||
comp_l = torch.tensor(-S / 2)
|
||||
comp_m = torch.tensor(S / 2)
|
||||
mask = torch.all(torch.concat([pts > comp_l, pts < comp_m], dim=-1), dim=-1)
|
||||
return mask
|
||||
|
||||
@staticmethod
|
||||
def from_params(
|
||||
pos: Tuple[float, float, float], rpy: Tuple[float, float, float], scale: Tuple[float, float, float]
|
||||
):
|
||||
"""Construct a box from position, rotation, and scale parameters."""
|
||||
R = torch.tensor(vtf.SO3.from_rpy_radians(rpy[0], rpy[1], rpy[2]).as_matrix())
|
||||
T = torch.tensor(pos)
|
||||
S = torch.tensor(scale)
|
||||
return OrientedBox(R=R, T=T, S=S)
|
||||
Reference in new issue
Block a user