diff --git a/nerfstudio/configs/base_config.py b/nerfstudio/configs/base_config.py index 53bc7b4c4..e4828fab6 100644 --- a/nerfstudio/configs/base_config.py +++ b/nerfstudio/configs/base_config.py @@ -228,6 +228,8 @@ class Config(PrintableConfig): """Which visualizer to use.""" data: Optional[Path] = None """Alias for --pipeline.datamanager.dataparser.data""" + wandb_name: Optional[str] = None + """Alias for --logging.wandb.name""" def is_viewer_enabled(self) -> bool: """Checks if a viewer is enabled.""" diff --git a/nerfstudio/fields/nerfacto_field.py b/nerfstudio/fields/nerfacto_field.py index da51f417a..ea215b517 100644 --- a/nerfstudio/fields/nerfacto_field.py +++ b/nerfstudio/fields/nerfacto_field.py @@ -99,6 +99,7 @@ class TCNNNerfactoField(Field): hidden_dim_transient: int = 64, appearance_embedding_dim: int = 32, transient_embedding_dim: int = 16, + use_appearance_embedding: bool = True, use_transient_embedding: bool = False, use_semantics: bool = False, num_semantic_classes: int = 100, @@ -114,12 +115,15 @@ class TCNNNerfactoField(Field): self.spatial_distortion = spatial_distortion self.num_images = num_images self.appearance_embedding_dim = appearance_embedding_dim - self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) self.use_average_appearance_embedding = use_average_appearance_embedding + self.use_appearance_embedding = use_appearance_embedding self.use_transient_embedding = use_transient_embedding self.use_semantics = use_semantics self.use_pred_normals = use_pred_normals + if self.use_appearance_embedding: + self.embedding_appearance = Embedding(self.num_images, self.appearance_embedding_dim) + base_res = 16 features_per_level = 2 growth_factor = np.exp((np.log(max_res) - np.log(base_res)) / (num_levels - 1)) @@ -208,8 +212,9 @@ class TCNNNerfactoField(Field): ) self.field_head_pred_normals = PredNormalsFieldHead(in_dim=self.mlp_pred_normals.n_output_dims) + appearance_embedding_dim = self.appearance_embedding_dim if self.use_appearance_embedding else 0 self.mlp_head = tcnn.Network( - n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + self.appearance_embedding_dim, + n_input_dims=self.direction_encoding.n_output_dims + self.geo_feat_dim + appearance_embedding_dim, n_output_dims=3, network_config={ "otype": "FullyFusedMLP", @@ -255,17 +260,18 @@ class TCNNNerfactoField(Field): outputs_shape = ray_samples.frustums.directions.shape[:-1] # appearance - if self.training: - embedded_appearance = self.embedding_appearance(camera_indices) - else: - if self.use_average_appearance_embedding: - embedded_appearance = torch.ones( - (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device - ) * self.embedding_appearance.mean(dim=0) + if self.use_appearance_embedding: + if self.training: + embedded_appearance = self.embedding_appearance(camera_indices) else: - embedded_appearance = torch.zeros( - (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device - ) + if self.use_average_appearance_embedding: + embedded_appearance = torch.ones( + (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + ) * self.embedding_appearance.mean(dim=0) + else: + embedded_appearance = torch.zeros( + (*directions.shape[:-1], self.appearance_embedding_dim), device=directions.device + ) # transients if self.use_transient_embedding and self.training: @@ -304,12 +310,14 @@ class TCNNNerfactoField(Field): x = self.mlp_pred_normals(pred_normals_inp).view(*outputs_shape, -1).to(directions) outputs[FieldHeadNames.PRED_NORMALS] = self.field_head_pred_normals(x) + h_list = [ + d, + density_embedding.view(-1, self.geo_feat_dim) + ] + if self.use_appearance_embedding: + h_list += [embedded_appearance.view(-1, self.appearance_embedding_dim)] h = torch.cat( - [ - d, - density_embedding.view(-1, self.geo_feat_dim), - embedded_appearance.view(-1, self.appearance_embedding_dim), - ], + h_list, dim=-1, ) rgb = self.mlp_head(h).view(*outputs_shape, -1).to(directions) diff --git a/nerfstudio/models/nerfacto.py b/nerfstudio/models/nerfacto.py index b1e772b03..6df0013c7 100644 --- a/nerfstudio/models/nerfacto.py +++ b/nerfstudio/models/nerfacto.py @@ -71,7 +71,7 @@ class NerfactoModelConfig(ModelConfig): """Whether to randomize the background color.""" num_levels: int = 16 """Number of levels of the hashmap for the base mlp.""" - max_res: int = 1024 + max_res: int = 2048 """Maximum resolution of the hashmap for the base mlp.""" log2_hashmap_size: int = 19 """Size of the hashmap for the base mlp""" @@ -104,6 +104,8 @@ class NerfactoModelConfig(ModelConfig): """Predicted normal loss multiplier.""" use_proposal_weight_anneal: bool = True """Whether to use proposal weight annealing.""" + use_appearance_embedding: bool = True + """Whether to use appearance embedding.""" use_average_appearance_embedding: bool = True """Whether to use average appearance embedding or zeros for inference.""" proposal_weights_anneal_slope: float = 10.0 @@ -140,6 +142,7 @@ class NerfactoModel(Model): spatial_distortion=scene_contraction, num_images=self.num_train_data, use_pred_normals=self.config.predict_normals, + use_appearance_embedding=self.config.use_appearance_embedding, use_average_appearance_embedding=self.config.use_average_appearance_embedding, ) diff --git a/nerfstudio/utils/writer.py b/nerfstudio/utils/writer.py index 76d7a3aa9..077c99b3f 100644 --- a/nerfstudio/utils/writer.py +++ b/nerfstudio/utils/writer.py @@ -198,7 +198,7 @@ def setup_event_writer(config: cfg.Config, log_dir: Path) -> None: """ using_event_writer = False if config.is_wandb_enabled(): - curr_writer = WandbWriter(log_dir=log_dir) + curr_writer = WandbWriter(log_dir=log_dir, wandb_name=config.wandb_name) EVENT_WRITERS.append(curr_writer) using_event_writer = True if config.is_tensorboard_enabled(): @@ -282,8 +282,11 @@ class TimeWriter: class WandbWriter(Writer): """WandDB Writer Class""" - def __init__(self, log_dir: Path): + def __init__(self, log_dir: Path, wandb_name: Optional[str] = None): wandb.init(project="nerfstudio-project", dir=str(log_dir), reinit=True) + if wandb_name: + wandb.run.name = wandb_name + wandb.run.save() def write_image(self, name: str, image: TensorType["H", "W", "C"], step: int) -> None: image = torch.permute(image, (2, 0, 1)) diff --git a/scripts/datasets/process_mipnerf360.py b/scripts/datasets/process_mipnerf360.py new file mode 100644 index 000000000..ef7996ce0 --- /dev/null +++ b/scripts/datasets/process_mipnerf360.py @@ -0,0 +1,66 @@ +""" +Process the Mipnerf360 dataset. +""" + +from nerfstudio.utils.scripts import run_command + +# # split = ["bicycle", "flowers", "garden", "stump", "treehill", "room", "counter", "kitchen", "bonsai"] +# splits = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"] + + +# # for processing data + +# for split in splits: +# print(f"Processing {split}...") +# working_dir = "/home/ethanweber/nerfactory/data/_prep/images/mipnerf360" +# cmd = f"ns-process-data images --data {working_dir}/{split}/images --output-dir {working_dir}/processed/{split}/" +# output = run_command(cmd, verbose=True) + +# for training + +# 1/8 of input images used in the paper = 0.125 -> 1 - this = 0.875 +# ns-train nerfacto --vis wandb --data data/nerfstudio/garden nerfstudio-data --downscale-factor 1 --train-split-percentage 0.875 +# X turn off pose optimization +# turn off appearance optimization +# increase hash map size. max_res 2048 +# ns-train nerfacto --vis wandb --data data/_prep/images/mipnerf360/processed/garden --pipeline.datamanager.camera-optimizer.mode off --trainer.steps-per-eval-all-images 5000 nerfstudio-data --downscale-factor 4 + + +# mipnerf 360 experiments + +splits = ["bicycle", "garden", "stump", "room", "counter", "kitchen", "bonsai"] # 7 splits +gpus = [0, 1, 3, 4, 5, 6, 7] + +commands = [] +for split, gpu in zip(splits, gpus): + wandb_name = f"mipnerf360-{split}" + command = " ".join( + ( + f"CUDA_VISIBLE_DEVICES={gpu}", + "ns-train nerfacto", + "--vis wandb", + f"--wandb-name {wandb_name}", + f"--data data/_prep/images/mipnerf360/processed/{split}", + "--pipeline.datamanager.camera-optimizer.mode off", + "--trainer.steps-per-eval-all-images 5000", + "--trainer.max-num-iterations 250000", + "--pipeline.model.use_appearance_embedding False", + "nerfstudio-data", + "--downscale-factor 4", + ) + ) + print(command) + commands.append(command) +# import subprocess +# import sys +# def f(cmd): +# out = subprocess.run(cmd, capture_output=False, shell=True, check=False) +# if out.returncode != 0: +# print(f"Error running command: {cmd}") +# sys.exit(1) +# if out.stdout is not None: +# return out.stdout.decode("utf-8") + +# from multiprocessing import Pool +# with Pool() as p: +# print(p.map(f, commands)) diff --git a/scripts/downloads/download_data.py b/scripts/downloads/download_data.py index c444991a9..b26886e89 100755 --- a/scripts/downloads/download_data.py +++ b/scripts/downloads/download_data.py @@ -235,7 +235,6 @@ class PhototourismDownload(DatasetDownload): shutil.rmtree(tmp_path) os.remove(download_path) - Commands = Union[ Annotated[BlenderDownload, tyro.conf.subcommand(name="blender")], Annotated[FriendsDownload, tyro.conf.subcommand(name="friends")],