diff --git a/projects-lerf/lerf/data/lerf_datamanager.py b/projects-lerf/lerf/data/lerf_datamanager.py index da627eb86..b9ace9050 100644 --- a/projects-lerf/lerf/data/lerf_datamanager.py +++ b/projects-lerf/lerf/data/lerf_datamanager.py @@ -85,17 +85,18 @@ class LERFDataManager(VanillaDataManager): # pylint: disable=abstract-method images = torch.cat(images) cache_dir = f"outputs/{self.config.dataparser.data.name}" - # clip_cache_path, dino_cache_path = self._search_cache(cache_dir) - clip_cache_path = osp.join(cache_dir, "clip") - dino_cache_path = osp.join(cache_dir, "dino.npy") + clip_cache_path = Path(osp.join(cache_dir, f"clip_{self.image_encoder.name}")) + dino_cache_path = Path(osp.join(cache_dir, "dino.npy")) + # NOTE: cache config is sensitive to list vs. tuple, because it checks for dict equality self.clip_interpolator = PyramidEmbeddingDataloader( image_list=images, device=self.device, cfg={ - "tile_size_range": (0.05, 0.5), + "tile_size_range": [0.05, 0.5], "tile_size_res": 7, "stride_scaler": 0.5, - "image_shape": tuple(images.shape[2:4]), + "image_shape": list(images.shape[2:4]), + "model_name": self.image_encoder.name }, cache_path=clip_cache_path, model=self.image_encoder, @@ -103,21 +104,10 @@ class LERFDataManager(VanillaDataManager): # pylint: disable=abstract-method self.dino_dataloader = DinoDataloader( image_list=images, device=self.device, - cfg={"image_shape": tuple(images.shape[2:4])}, + cfg={"image_shape": list(images.shape[2:4])}, cache_path=dino_cache_path, ) - # def _search_cache(self, cache_path, prop_dict): - # if osp.exists(cache_path): - # # load yaml file - # with open(cache_path, "r") as f: - # cache = yaml.load(f, Loader=yaml.FullLoader) - # for k, v in cache.items(): - # if k in self.config: - # self.config[k] = v - # else: - # ... - def next_train(self, step: int) -> Tuple[RayBundle, Dict]: """Returns the next batch of data from the train dataloader.""" self.train_count += 1 diff --git a/projects-lerf/lerf/data/utils/feature_dataloader.py b/projects-lerf/lerf/data/utils/feature_dataloader.py index 2b98f6b10..3150fbaf5 100644 --- a/projects-lerf/lerf/data/utils/feature_dataloader.py +++ b/projects-lerf/lerf/data/utils/feature_dataloader.py @@ -2,6 +2,7 @@ import os import typing from abc import ABC, ABCMeta, abstractmethod from pathlib import Path +import json import numpy as np import torch @@ -30,14 +31,24 @@ class FeatureDataloader(ABC): pass def load(self): + cache_info_path = self.cache_path.with_suffix(".info") + if not cache_info_path.exists(): + raise FileNotFoundError + with open(cache_info_path, "r") as f: + cfg = json.loads(f.read()) + if cfg != self.cfg: + raise ValueError("Config mismatch") self.data = torch.from_numpy(np.load(self.cache_path)) def save(self): + cache_info_path = self.cache_path.with_suffix(".info") + with open(cache_info_path, "w") as f: + f.write(json.dumps(self.cfg)) np.save(self.cache_path, self.data) def try_load(self, img_list: torch.Tensor): try: self.load() - except FileNotFoundError: + except (FileNotFoundError, ValueError): self.create(img_list) self.save() \ No newline at end of file diff --git a/projects-lerf/lerf/data/utils/patch_embedding_dataloader.py b/projects-lerf/lerf/data/utils/patch_embedding_dataloader.py index 4f5182fe3..b2f0e0f6b 100644 --- a/projects-lerf/lerf/data/utils/patch_embedding_dataloader.py +++ b/projects-lerf/lerf/data/utils/patch_embedding_dataloader.py @@ -17,6 +17,7 @@ class PatchEmbeddingDataloader(FeatureDataloader): assert "tile_ratio" in cfg assert "stride_ratio" in cfg assert "image_shape" in cfg + assert "model_name" in cfg self.kernel_size = int(cfg["image_shape"][0] * cfg["tile_ratio"]) self.stride = int(self.kernel_size * cfg["stride_ratio"]) diff --git a/projects-lerf/lerf/data/utils/pyramid_embedding_dataloader.py b/projects-lerf/lerf/data/utils/pyramid_embedding_dataloader.py index 08f97a446..0a8d2b52f 100644 --- a/projects-lerf/lerf/data/utils/pyramid_embedding_dataloader.py +++ b/projects-lerf/lerf/data/utils/pyramid_embedding_dataloader.py @@ -6,6 +6,8 @@ from lerf.data.utils.feature_dataloader import FeatureDataloader from lerf.data.utils.patch_embedding_dataloader import PatchEmbeddingDataloader from lerf.encoders.image_encoder import BaseImageEncoder from tqdm import tqdm +from pathlib import Path +import json class PyramidEmbeddingDataloader(FeatureDataloader): @@ -21,9 +23,10 @@ class PyramidEmbeddingDataloader(FeatureDataloader): assert "tile_size_res" in cfg assert "stride_scaler" in cfg assert "image_shape" in cfg + assert "model_name" in cfg - cfg["tile_sizes"] = torch.linspace(*cfg["tile_size_range"], cfg["tile_size_res"]).to(device) - cfg["strider_scaler_list"] = [self._stride_scaler(tr.item(), cfg["stride_scaler"]) for tr in cfg["tile_sizes"]] + self.tile_sizes = torch.linspace(*cfg["tile_size_range"], cfg["tile_size_res"]).to(device) + self.strider_scaler_list = [self._stride_scaler(tr.item(), cfg["stride_scaler"]) for tr in self.tile_sizes] self.model = model self.embed_size = self.model.embedding_dim @@ -41,38 +44,58 @@ class PyramidEmbeddingDataloader(FeatureDataloader): def load(self): # don't create anything, PatchEmbeddingDataloader will create itself + cache_info_path = self.cache_path.with_suffix(".info") + + # check if cache exists + if not cache_info_path.exists(): + raise FileNotFoundError + + # if config is different, remove all cached content + with open(cache_info_path, "r") as f: + cfg = json.loads(f.read()) + if cfg != self.cfg: + for f in os.listdir(self.cache_path): + os.remove(os.path.join(self.cache_path, f)) + raise ValueError("Config mismatch") + raise FileNotFoundError # trigger create def create(self, image_list): os.makedirs(self.cache_path, exist_ok=True) - for i, tr in enumerate(tqdm(self.cfg["tile_sizes"], desc="Scales")): - stride_scaler = self.cfg["strider_scaler_list"][i] + for i, tr in enumerate(tqdm(self.tile_sizes, desc="Scales")): + stride_scaler = self.strider_scaler_list[i] self.data_dict[i] = PatchEmbeddingDataloader( - cfg={"tile_ratio": tr.item(), "stride_ratio": stride_scaler, "image_shape": self.cfg["image_shape"]}, + cfg={ + "tile_ratio": tr.item(), + "stride_ratio": stride_scaler, + "image_shape": self.cfg["image_shape"], + "model_name": self.cfg["model_name"] + }, device=self.device, model=self.model, image_list=image_list, - cache_path=f"{self.cache_path}/level_{i}.npy", + cache_path=Path(f"{self.cache_path}/level_{i}.npy"), ) def save(self): + cache_info_path = self.cache_path.with_suffix(".info") + with open(cache_info_path, "w") as f: + f.write(json.dumps(self.cfg)) # don't save anything, PatchEmbeddingDataloader will save itself pass def _random_scales(self, img_points): # img_points: (B, 3) # (img_ind, x, y) # return: (B, 512), some random scale (between 0, 1) - tile_sizes = self.cfg["tile_sizes"] - - random_scale_bin = torch.randint(tile_sizes.shape[0] - 1, size=(img_points.shape[0],), device=self.device) + random_scale_bin = torch.randint(self.tile_sizes.shape[0] - 1, size=(img_points.shape[0],), device=self.device) random_scale_weight = torch.rand(img_points.shape[0], dtype=torch.float16, device=self.device) - stepsize = (tile_sizes[1] - tile_sizes[0]) / (tile_sizes[-1] - tile_sizes[0]) + stepsize = (self.tile_sizes[1] - self.tile_sizes[0]) / (self.tile_sizes[-1] - self.tile_sizes[0]) bottom_interp = torch.zeros((img_points.shape[0], self.embed_size), dtype=torch.float16, device=self.device) top_interp = torch.zeros((img_points.shape[0], self.embed_size), dtype=torch.float16, device=self.device) - for i in range(len(tile_sizes) - 1): + for i in range(len(self.tile_sizes) - 1): bottom_interp[random_scale_bin == i] = self.data_dict[i](img_points[random_scale_bin == i]) top_interp[random_scale_bin == i] = self.data_dict[i + 1](img_points[random_scale_bin == i]) @@ -82,12 +105,10 @@ class PyramidEmbeddingDataloader(FeatureDataloader): ) def _uniform_scales(self, img_points, scale): - tile_sizes = self.cfg["tile_sizes"] - scale_bin = torch.floor( - (scale - tile_sizes[0]) / (tile_sizes[-1] - tile_sizes[0]) * (tile_sizes.shape[0] - 1) + (scale - self.tile_sizes[0]) / (self.tile_sizes[-1] - self.tile_sizes[0]) * (self.tile_sizes.shape[0] - 1) ).to(torch.int64) - scale_weight = (scale - tile_sizes[scale_bin]) / (tile_sizes[scale_bin + 1] - tile_sizes[scale_bin]) + scale_weight = (scale - self.tile_sizes[scale_bin]) / (self.tile_sizes[scale_bin + 1] - self.tile_sizes[scale_bin]) interp_lst = torch.stack([interp(img_points) for interp in self.data_dict.values()]) point_inds = torch.arange(img_points.shape[0]) interp = torch.lerp( diff --git a/projects-lerf/lerf/encoders/clip_encoder.py b/projects-lerf/lerf/encoders/clip_encoder.py index 097e08dd2..eb238346a 100644 --- a/projects-lerf/lerf/encoders/clip_encoder.py +++ b/projects-lerf/lerf/encoders/clip_encoder.py @@ -14,9 +14,6 @@ from lerf.encoders.image_encoder import BaseImageEncoder, BaseImageEncoderConfig @dataclass class CLIPNetworkConfig(BaseImageEncoderConfig): _target: Type = field(default_factory=lambda: CLIPNetwork) - name: str = "clip" - description: str = "OpenAI CLIP network, images and text" - """target class to instantiate""" clip_model_type: str = "ViT-B-16" clip_n_dims: int = 512 negatives: Tuple[str] = ("object", "things", "stuff", "texture") @@ -56,6 +53,10 @@ class CLIPNetwork(BaseImageEncoder): self.pos_embeds.shape[1] == self.clip_n_dims ), "Embedding dimensionality must match the model dimensionality" + @property + def name(self) -> str: + return "clip_{}".format(self.config.clip_model_type) + @property def embedding_dim(self) -> int: return self.config.clip_n_dims diff --git a/projects-lerf/lerf/encoders/image_encoder.py b/projects-lerf/lerf/encoders/image_encoder.py index 08496f8b8..cc2851f6e 100644 --- a/projects-lerf/lerf/encoders/image_encoder.py +++ b/projects-lerf/lerf/encoders/image_encoder.py @@ -9,12 +9,14 @@ from nerfstudio.configs import base_config as cfg @dataclass class BaseImageEncoderConfig(cfg.InstantiateConfig): _target: Type = field(default_factory=lambda: BaseImageEncoder) - name: str = "" - """name of the encoder""" - description: str = "" - """description of the encoder""" class BaseImageEncoder: + @abstractproperty + def name(self) -> str: + """ + returns the name of the encoder + """ + @abstractproperty def embedding_dim(self) -> int: """ diff --git a/projects-lerf/lerf/encoders/openclip_encoder.py b/projects-lerf/lerf/encoders/openclip_encoder.py index 1a6dbb29e..6f0631751 100644 --- a/projects-lerf/lerf/encoders/openclip_encoder.py +++ b/projects-lerf/lerf/encoders/openclip_encoder.py @@ -15,9 +15,6 @@ from lerf.encoders.image_encoder import BaseImageEncoder, BaseImageEncoderConfig @dataclass class OpenCLIPNetworkConfig(BaseImageEncoderConfig): _target: Type = field(default_factory=lambda: OpenCLIPNetwork) - name: str = "openclip_vit_b_16" - description: str = "OpenCLIP network, images and text" - """target class to instantiate""" clip_model_type: str = "ViT-B-16" clip_model_pretrained: str = "laion2b_s34b_b88k" clip_n_dims: int = 512 @@ -62,6 +59,10 @@ class OpenCLIPNetwork(BaseImageEncoder): self.pos_embeds.shape[1] == self.clip_n_dims ), "Embedding dimensionality must match the model dimensionality" + @property + def name(self) -> str: + return "openclip_{}_{}".format(self.config.clip_model_type, self.config.clip_model_pretrained) + @property def embedding_dim(self) -> int: return self.config.clip_n_dims