mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 22:56:03 +08:00
Monodepth Regularization (#2284)
* monodepth implemented * device * pair pixel sampler * docstring * removing unused imports * fix bugs * cache in cache dir, no longer edit json file * moving to depth nerfacto * fixed bug using median depth by accident * removing depthdataset from nerfacto * added pixelsamplerconfig * added equirectangular warning back * bug fix * delete sample_config.py + small fixes * rename to pixel_sampler * cleaning * test * passing num_rays_per_batch properly * fixed transforms typo * renaming in nerfacto * remove depth files from test data * removed defaulting to base pixel sampler with mask * adding patch_size back --------- Co-authored-by: AdamRashid96 <abrashid@berkeley.edu> Co-authored-by: Matthew Tancik <tancik@berkeley.edu>
This commit is contained in:
8 files changed
+293
-89
No files matched your search
@@ -18,31 +18,95 @@ Depth dataset.
|
||||
|
||||
from typing import Dict
|
||||
|
||||
import numpy as np
|
||||
|
||||
from nerfstudio.data.dataparsers.base_dataparser import DataparserOutputs
|
||||
from nerfstudio.model_components import losses
|
||||
from nerfstudio.data.datasets.base_dataset import InputDataset
|
||||
from nerfstudio.data.utils.data_utils import get_depth_image_from_path
|
||||
from nerfstudio.utils.rich_utils import CONSOLE
|
||||
|
||||
from typing import Union
|
||||
from PIL import Image
|
||||
import torch
|
||||
from rich.progress import track
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
|
||||
class DepthDataset(InputDataset):
|
||||
"""Dataset that returns images and depths.
|
||||
"""Dataset that returns images and depths. If no depths are found, then we generate them with Zoe Depth.
|
||||
|
||||
Args:
|
||||
dataparser_outputs: description of where and how to read input images.
|
||||
scale_factor: The scaling factor for the dataparser outputs.
|
||||
"""
|
||||
|
||||
exclude_batch_keys_from_device = InputDataset.exclude_batch_keys_from_device + ["depth_image"]
|
||||
|
||||
def __init__(self, dataparser_outputs: DataparserOutputs, scale_factor: float = 1.0):
|
||||
super().__init__(dataparser_outputs, scale_factor)
|
||||
assert (
|
||||
"depth_filenames" in dataparser_outputs.metadata.keys()
|
||||
and dataparser_outputs.metadata["depth_filenames"] is not None
|
||||
)
|
||||
# if there are no depth images than we want to generate them all with zoe depth
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
if len(dataparser_outputs.image_filenames) > 0 and (
|
||||
"depth_filenames" not in dataparser_outputs.metadata.keys()
|
||||
or dataparser_outputs.metadata["depth_filenames"] is None
|
||||
):
|
||||
losses.DEPTH_METRIC = 0
|
||||
CONSOLE.print("[bold yellow] No depth data found!")
|
||||
cache = dataparser_outputs.image_filenames[0].parent / "depths.npy"
|
||||
if cache.exists():
|
||||
CONSOLE.print("[bold yellow] Loading cache!")
|
||||
# load all the depths
|
||||
self.depths = np.load(cache)
|
||||
self.depths = torch.from_numpy(self.depths).to(device)
|
||||
else:
|
||||
depth_tensors = []
|
||||
transforms = self._find_transform(dataparser_outputs.image_filenames[0])
|
||||
data = dataparser_outputs.image_filenames[0].parent
|
||||
if transforms is not None:
|
||||
meta = json.load(open(transforms, "r"))
|
||||
frames = meta["frames"]
|
||||
filenames = [data / frames[j]["file_path"].split("/")[-1] for j in range(len(frames))]
|
||||
else:
|
||||
meta = None
|
||||
frames = None
|
||||
filenames = dataparser_outputs.image_filenames
|
||||
|
||||
repo = "isl-org/ZoeDepth"
|
||||
self.zoe = torch.compile(torch.hub.load(repo, "ZoeD_NK", pretrained=True).to(device))
|
||||
|
||||
for i in track(range(len(filenames)), description="Generating depth images"):
|
||||
image_filename = filenames[i]
|
||||
pil_image = Image.open(image_filename)
|
||||
image = np.array(pil_image, dtype="uint8") # shape is (h, w) or (h, w, 3 or 4)
|
||||
if len(image.shape) == 2:
|
||||
image = image[:, :, None].repeat(3, axis=2)
|
||||
image = torch.from_numpy(image.astype("float32") / 255.0)
|
||||
|
||||
with torch.no_grad():
|
||||
image = torch.permute(image, (2, 0, 1)).unsqueeze(0).to(device)
|
||||
if image.shape[1] == 4:
|
||||
image = image[:, :3, :, :]
|
||||
depth_tensor = self.zoe.infer(image).squeeze().unsqueeze(-1)
|
||||
|
||||
depth_tensors.append(depth_tensor)
|
||||
|
||||
self.depths = torch.stack(depth_tensors)
|
||||
np.save(cache, self.depths.cpu().numpy())
|
||||
dataparser_outputs.metadata["depth_filenames"] = None
|
||||
dataparser_outputs.metadata["depth_unit_scale_factor"] = 1.0
|
||||
self.metadata["depth_filenames"] = None
|
||||
self.metadata["depth_unit_scale_factor"] = 1.0
|
||||
|
||||
else:
|
||||
losses.DEPTH_METRIC = 0
|
||||
|
||||
self.depth_filenames = self.metadata["depth_filenames"]
|
||||
self.depth_unit_scale_factor = self.metadata["depth_unit_scale_factor"]
|
||||
|
||||
def get_metadata(self, data: Dict) -> Dict:
|
||||
if self.depth_filenames is None:
|
||||
return {"depth_image": self.depths[data["image_idx"]]}
|
||||
|
||||
filepath = self.depth_filenames[data["image_idx"]]
|
||||
height = int(self._dataparser_outputs.cameras.height[data["image_idx"]])
|
||||
width = int(self._dataparser_outputs.cameras.width[data["image_idx"]])
|
||||
@@ -54,3 +118,11 @@ class DepthDataset(InputDataset):
|
||||
)
|
||||
|
||||
return {"depth_image": depth_image}
|
||||
|
||||
def _find_transform(self, image_path: Path) -> Union[Path, None]:
|
||||
while image_path.parent != image_path:
|
||||
transform_path = image_path.parent / "transforms.json"
|
||||
if transform_path.exists():
|
||||
return transform_path
|
||||
image_path = image_path.parent
|
||||
return None
|
||||
Reference in new issue
Block a user