mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-16 12:53:44 +08:00
Added cache capabilities for data
This commit is contained in:
1 parent
5952a692eb
commit
d8b2462e50
7 files changed
+70
-43
No files matched your search
@@ -85,17 +85,18 @@ class LERFDataManager(VanillaDataManager): # pylint: disable=abstract-method
|
||||
images = torch.cat(images)
|
||||
|
||||
cache_dir = f"outputs/{self.config.dataparser.data.name}"
|
||||
# clip_cache_path, dino_cache_path = self._search_cache(cache_dir)
|
||||
clip_cache_path = osp.join(cache_dir, "clip")
|
||||
dino_cache_path = osp.join(cache_dir, "dino.npy")
|
||||
clip_cache_path = Path(osp.join(cache_dir, f"clip_{self.image_encoder.name}"))
|
||||
dino_cache_path = Path(osp.join(cache_dir, "dino.npy"))
|
||||
# NOTE: cache config is sensitive to list vs. tuple, because it checks for dict equality
|
||||
self.clip_interpolator = PyramidEmbeddingDataloader(
|
||||
image_list=images,
|
||||
device=self.device,
|
||||
cfg={
|
||||
"tile_size_range": (0.05, 0.5),
|
||||
"tile_size_range": [0.05, 0.5],
|
||||
"tile_size_res": 7,
|
||||
"stride_scaler": 0.5,
|
||||
"image_shape": tuple(images.shape[2:4]),
|
||||
"image_shape": list(images.shape[2:4]),
|
||||
"model_name": self.image_encoder.name
|
||||
},
|
||||
cache_path=clip_cache_path,
|
||||
model=self.image_encoder,
|
||||
@@ -103,21 +104,10 @@ class LERFDataManager(VanillaDataManager): # pylint: disable=abstract-method
|
||||
self.dino_dataloader = DinoDataloader(
|
||||
image_list=images,
|
||||
device=self.device,
|
||||
cfg={"image_shape": tuple(images.shape[2:4])},
|
||||
cfg={"image_shape": list(images.shape[2:4])},
|
||||
cache_path=dino_cache_path,
|
||||
)
|
||||
|
||||
# def _search_cache(self, cache_path, prop_dict):
|
||||
# if osp.exists(cache_path):
|
||||
# # load yaml file
|
||||
# with open(cache_path, "r") as f:
|
||||
# cache = yaml.load(f, Loader=yaml.FullLoader)
|
||||
# for k, v in cache.items():
|
||||
# if k in self.config:
|
||||
# self.config[k] = v
|
||||
# else:
|
||||
# ...
|
||||
|
||||
def next_train(self, step: int) -> Tuple[RayBundle, Dict]:
|
||||
"""Returns the next batch of data from the train dataloader."""
|
||||
self.train_count += 1
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
import typing
|
||||
from abc import ABC, ABCMeta, abstractmethod
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -30,14 +31,24 @@ class FeatureDataloader(ABC):
|
||||
pass
|
||||
|
||||
def load(self):
|
||||
cache_info_path = self.cache_path.with_suffix(".info")
|
||||
if not cache_info_path.exists():
|
||||
raise FileNotFoundError
|
||||
with open(cache_info_path, "r") as f:
|
||||
cfg = json.loads(f.read())
|
||||
if cfg != self.cfg:
|
||||
raise ValueError("Config mismatch")
|
||||
self.data = torch.from_numpy(np.load(self.cache_path))
|
||||
|
||||
def save(self):
|
||||
cache_info_path = self.cache_path.with_suffix(".info")
|
||||
with open(cache_info_path, "w") as f:
|
||||
f.write(json.dumps(self.cfg))
|
||||
np.save(self.cache_path, self.data)
|
||||
|
||||
def try_load(self, img_list: torch.Tensor):
|
||||
try:
|
||||
self.load()
|
||||
except FileNotFoundError:
|
||||
except (FileNotFoundError, ValueError):
|
||||
self.create(img_list)
|
||||
self.save()
|
||||
@@ -17,6 +17,7 @@ class PatchEmbeddingDataloader(FeatureDataloader):
|
||||
assert "tile_ratio" in cfg
|
||||
assert "stride_ratio" in cfg
|
||||
assert "image_shape" in cfg
|
||||
assert "model_name" in cfg
|
||||
|
||||
self.kernel_size = int(cfg["image_shape"][0] * cfg["tile_ratio"])
|
||||
self.stride = int(self.kernel_size * cfg["stride_ratio"])
|
||||
|
||||
@@ -6,6 +6,8 @@ from lerf.data.utils.feature_dataloader import FeatureDataloader
|
||||
from lerf.data.utils.patch_embedding_dataloader import PatchEmbeddingDataloader
|
||||
from lerf.encoders.image_encoder import BaseImageEncoder
|
||||
from tqdm import tqdm
|
||||
from pathlib import Path
|
||||
import json
|
||||
|
||||
|
||||
class PyramidEmbeddingDataloader(FeatureDataloader):
|
||||
@@ -21,9 +23,10 @@ class PyramidEmbeddingDataloader(FeatureDataloader):
|
||||
assert "tile_size_res" in cfg
|
||||
assert "stride_scaler" in cfg
|
||||
assert "image_shape" in cfg
|
||||
assert "model_name" in cfg
|
||||
|
||||
cfg["tile_sizes"] = torch.linspace(*cfg["tile_size_range"], cfg["tile_size_res"]).to(device)
|
||||
cfg["strider_scaler_list"] = [self._stride_scaler(tr.item(), cfg["stride_scaler"]) for tr in cfg["tile_sizes"]]
|
||||
self.tile_sizes = torch.linspace(*cfg["tile_size_range"], cfg["tile_size_res"]).to(device)
|
||||
self.strider_scaler_list = [self._stride_scaler(tr.item(), cfg["stride_scaler"]) for tr in self.tile_sizes]
|
||||
|
||||
self.model = model
|
||||
self.embed_size = self.model.embedding_dim
|
||||
@@ -41,38 +44,58 @@ class PyramidEmbeddingDataloader(FeatureDataloader):
|
||||
|
||||
def load(self):
|
||||
# don't create anything, PatchEmbeddingDataloader will create itself
|
||||
cache_info_path = self.cache_path.with_suffix(".info")
|
||||
|
||||
# check if cache exists
|
||||
if not cache_info_path.exists():
|
||||
raise FileNotFoundError
|
||||
|
||||
# if config is different, remove all cached content
|
||||
with open(cache_info_path, "r") as f:
|
||||
cfg = json.loads(f.read())
|
||||
if cfg != self.cfg:
|
||||
for f in os.listdir(self.cache_path):
|
||||
os.remove(os.path.join(self.cache_path, f))
|
||||
raise ValueError("Config mismatch")
|
||||
|
||||
raise FileNotFoundError # trigger create
|
||||
|
||||
def create(self, image_list):
|
||||
os.makedirs(self.cache_path, exist_ok=True)
|
||||
for i, tr in enumerate(tqdm(self.cfg["tile_sizes"], desc="Scales")):
|
||||
stride_scaler = self.cfg["strider_scaler_list"][i]
|
||||
for i, tr in enumerate(tqdm(self.tile_sizes, desc="Scales")):
|
||||
stride_scaler = self.strider_scaler_list[i]
|
||||
self.data_dict[i] = PatchEmbeddingDataloader(
|
||||
cfg={"tile_ratio": tr.item(), "stride_ratio": stride_scaler, "image_shape": self.cfg["image_shape"]},
|
||||
cfg={
|
||||
"tile_ratio": tr.item(),
|
||||
"stride_ratio": stride_scaler,
|
||||
"image_shape": self.cfg["image_shape"],
|
||||
"model_name": self.cfg["model_name"]
|
||||
},
|
||||
device=self.device,
|
||||
model=self.model,
|
||||
image_list=image_list,
|
||||
cache_path=f"{self.cache_path}/level_{i}.npy",
|
||||
cache_path=Path(f"{self.cache_path}/level_{i}.npy"),
|
||||
)
|
||||
|
||||
def save(self):
|
||||
cache_info_path = self.cache_path.with_suffix(".info")
|
||||
with open(cache_info_path, "w") as f:
|
||||
f.write(json.dumps(self.cfg))
|
||||
# don't save anything, PatchEmbeddingDataloader will save itself
|
||||
pass
|
||||
|
||||
def _random_scales(self, img_points):
|
||||
# img_points: (B, 3) # (img_ind, x, y)
|
||||
# return: (B, 512), some random scale (between 0, 1)
|
||||
tile_sizes = self.cfg["tile_sizes"]
|
||||
|
||||
random_scale_bin = torch.randint(tile_sizes.shape[0] - 1, size=(img_points.shape[0],), device=self.device)
|
||||
random_scale_bin = torch.randint(self.tile_sizes.shape[0] - 1, size=(img_points.shape[0],), device=self.device)
|
||||
random_scale_weight = torch.rand(img_points.shape[0], dtype=torch.float16, device=self.device)
|
||||
|
||||
stepsize = (tile_sizes[1] - tile_sizes[0]) / (tile_sizes[-1] - tile_sizes[0])
|
||||
stepsize = (self.tile_sizes[1] - self.tile_sizes[0]) / (self.tile_sizes[-1] - self.tile_sizes[0])
|
||||
|
||||
bottom_interp = torch.zeros((img_points.shape[0], self.embed_size), dtype=torch.float16, device=self.device)
|
||||
top_interp = torch.zeros((img_points.shape[0], self.embed_size), dtype=torch.float16, device=self.device)
|
||||
|
||||
for i in range(len(tile_sizes) - 1):
|
||||
for i in range(len(self.tile_sizes) - 1):
|
||||
bottom_interp[random_scale_bin == i] = self.data_dict[i](img_points[random_scale_bin == i])
|
||||
top_interp[random_scale_bin == i] = self.data_dict[i + 1](img_points[random_scale_bin == i])
|
||||
|
||||
@@ -82,12 +105,10 @@ class PyramidEmbeddingDataloader(FeatureDataloader):
|
||||
)
|
||||
|
||||
def _uniform_scales(self, img_points, scale):
|
||||
tile_sizes = self.cfg["tile_sizes"]
|
||||
|
||||
scale_bin = torch.floor(
|
||||
(scale - tile_sizes[0]) / (tile_sizes[-1] - tile_sizes[0]) * (tile_sizes.shape[0] - 1)
|
||||
(scale - self.tile_sizes[0]) / (self.tile_sizes[-1] - self.tile_sizes[0]) * (self.tile_sizes.shape[0] - 1)
|
||||
).to(torch.int64)
|
||||
scale_weight = (scale - tile_sizes[scale_bin]) / (tile_sizes[scale_bin + 1] - tile_sizes[scale_bin])
|
||||
scale_weight = (scale - self.tile_sizes[scale_bin]) / (self.tile_sizes[scale_bin + 1] - self.tile_sizes[scale_bin])
|
||||
interp_lst = torch.stack([interp(img_points) for interp in self.data_dict.values()])
|
||||
point_inds = torch.arange(img_points.shape[0])
|
||||
interp = torch.lerp(
|
||||
|
||||
@@ -14,9 +14,6 @@ from lerf.encoders.image_encoder import BaseImageEncoder, BaseImageEncoderConfig
|
||||
@dataclass
|
||||
class CLIPNetworkConfig(BaseImageEncoderConfig):
|
||||
_target: Type = field(default_factory=lambda: CLIPNetwork)
|
||||
name: str = "clip"
|
||||
description: str = "OpenAI CLIP network, images and text"
|
||||
"""target class to instantiate"""
|
||||
clip_model_type: str = "ViT-B-16"
|
||||
clip_n_dims: int = 512
|
||||
negatives: Tuple[str] = ("object", "things", "stuff", "texture")
|
||||
@@ -56,6 +53,10 @@ class CLIPNetwork(BaseImageEncoder):
|
||||
self.pos_embeds.shape[1] == self.clip_n_dims
|
||||
), "Embedding dimensionality must match the model dimensionality"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "clip_{}".format(self.config.clip_model_type)
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return self.config.clip_n_dims
|
||||
|
||||
@@ -9,12 +9,14 @@ from nerfstudio.configs import base_config as cfg
|
||||
@dataclass
|
||||
class BaseImageEncoderConfig(cfg.InstantiateConfig):
|
||||
_target: Type = field(default_factory=lambda: BaseImageEncoder)
|
||||
name: str = ""
|
||||
"""name of the encoder"""
|
||||
description: str = ""
|
||||
"""description of the encoder"""
|
||||
|
||||
class BaseImageEncoder:
|
||||
@abstractproperty
|
||||
def name(self) -> str:
|
||||
"""
|
||||
returns the name of the encoder
|
||||
"""
|
||||
|
||||
@abstractproperty
|
||||
def embedding_dim(self) -> int:
|
||||
"""
|
||||
|
||||
@@ -15,9 +15,6 @@ from lerf.encoders.image_encoder import BaseImageEncoder, BaseImageEncoderConfig
|
||||
@dataclass
|
||||
class OpenCLIPNetworkConfig(BaseImageEncoderConfig):
|
||||
_target: Type = field(default_factory=lambda: OpenCLIPNetwork)
|
||||
name: str = "openclip_vit_b_16"
|
||||
description: str = "OpenCLIP network, images and text"
|
||||
"""target class to instantiate"""
|
||||
clip_model_type: str = "ViT-B-16"
|
||||
clip_model_pretrained: str = "laion2b_s34b_b88k"
|
||||
clip_n_dims: int = 512
|
||||
@@ -62,6 +59,10 @@ class OpenCLIPNetwork(BaseImageEncoder):
|
||||
self.pos_embeds.shape[1] == self.clip_n_dims
|
||||
), "Embedding dimensionality must match the model dimensionality"
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "openclip_{}_{}".format(self.config.clip_model_type, self.config.clip_model_pretrained)
|
||||
|
||||
@property
|
||||
def embedding_dim(self) -> int:
|
||||
return self.config.clip_n_dims
|
||||
|
||||
Reference in new issue
Block a user