From 73bffe058f2b64edfd25d8a4428ae9a5127f6664 Mon Sep 17 00:00:00 2001 From: Terrance Wang Date: Fri, 20 Jan 2023 12:00:51 -0800 Subject: [PATCH] Fix tensorf chkpt loading (#1266) * setting tensorf default from wandb to viewer * adding functionality to adjust model depending on the step of the checkpoint that is loaded * cleaning code * bug fix * fixing style --- nerfstudio/engine/trainer.py | 2 +- nerfstudio/models/base_model.py | 8 ++++++++ nerfstudio/models/tensorf.py | 16 +++++++++++++++- nerfstudio/pipelines/base_pipeline.py | 7 +++++-- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/nerfstudio/engine/trainer.py b/nerfstudio/engine/trainer.py index 49879c31d..b8aace10f 100644 --- a/nerfstudio/engine/trainer.py +++ b/nerfstudio/engine/trainer.py @@ -315,7 +315,7 @@ class Trainer: loaded_state = torch.load(load_path, map_location="cpu") self._start_step = loaded_state["step"] + 1 # load the checkpoints for pipeline, optimizers, and gradient scalar - self.pipeline.load_pipeline(loaded_state["pipeline"]) + self.pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"]) self.optimizers.load_optimizers(loaded_state["optimizers"]) self.grad_scaler.load_state_dict(loaded_state["scalers"]) CONSOLE.print(f"done loading checkpoint from {load_path}") diff --git a/nerfstudio/models/base_model.py b/nerfstudio/models/base_model.py index 3de53c153..3cc5fa2ec 100644 --- a/nerfstudio/models/base_model.py +++ b/nerfstudio/models/base_model.py @@ -210,3 +210,11 @@ class Model(nn.Module): """ state = {key.replace("module.", ""): value for key, value in loaded_state["model"].items()} self.load_state_dict(state) # type: ignore + + def update_to_step(self, step: int) -> None: + """Called when loading a model from a checkpoint. Sets any model parameters that change over + training to the correct value, based on the training step of the checkpoint. + + Args: + step: training step of the loaded checkpoint + """ diff --git a/nerfstudio/models/tensorf.py b/nerfstudio/models/tensorf.py index 11e6e24bc..60fec9706 100644 --- a/nerfstudio/models/tensorf.py +++ b/nerfstudio/models/tensorf.py @@ -114,7 +114,8 @@ class TensoRFModel(Model): def reinitialize_optimizer( self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument ): - resolution = self.upsampling_steps.pop(0) + index = self.upsampling_iters.index(step) + resolution = self.upsampling_steps[index] # upsample the position and direction grids self.field.density_encoding.upsample_grid(resolution) @@ -143,6 +144,19 @@ class TensoRFModel(Model): ] return callbacks + def update_to_step(self, step: int) -> None: + if step < self.upsampling_iters[0]: + return + + new_iters = list(self.upsampling_iters) + [step + 1] + new_iters.sort() + + index = new_iters.index(step + 1) + new_grid_resolution = self.upsampling_steps[index - 1] + + self.field.density_encoding.upsample_grid(new_grid_resolution) # type: ignore + self.field.color_encoding.upsample_grid(new_grid_resolution) # type: ignore + def populate_modules(self): """Set the fields and modules""" super().populate_modules() diff --git a/nerfstudio/pipelines/base_pipeline.py b/nerfstudio/pipelines/base_pipeline.py index 343cf5104..9d062e02f 100644 --- a/nerfstudio/pipelines/base_pipeline.py +++ b/nerfstudio/pipelines/base_pipeline.py @@ -158,11 +158,12 @@ class Pipeline(nn.Module): def get_average_eval_image_metrics(self, step: Optional[int] = None): """Iterate over all the images in the eval dataset and get the average.""" - def load_pipeline(self, loaded_state: Dict[str, Any]) -> None: + def load_pipeline(self, loaded_state: Dict[str, Any], step: int) -> None: """Load the checkpoint from the given path Args: loaded_state: pre-trained model state dict + step: training step of the loaded checkpoint """ def get_training_callbacks( @@ -352,13 +353,15 @@ class VanillaPipeline(Pipeline): self.train() return metrics_dict - def load_pipeline(self, loaded_state: Dict[str, Any]) -> None: + def load_pipeline(self, loaded_state: Dict[str, Any], step: int) -> None: """Load the checkpoint from the given path Args: loaded_state: pre-trained model state dict + step: training step of the loaded checkpoint """ state = {key.replace("module.", ""): value for key, value in loaded_state.items()} + self._model.update_to_step(step) self.load_state_dict(state, strict=False) def get_training_callbacks(