mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-21 15:15:43 +08:00
Fix tensorf chkpt loading (#1266)
* setting tensorf default from wandb to viewer * adding functionality to adjust model depending on the step of the checkpoint that is loaded * cleaning code * bug fix * fixing style
This commit is contained in:
1 parent
9eb880aa91
commit
73bffe058f
4 files changed
+29
-4
No files matched your search
@@ -315,7 +315,7 @@ class Trainer:
|
||||
loaded_state = torch.load(load_path, map_location="cpu")
|
||||
self._start_step = loaded_state["step"] + 1
|
||||
# load the checkpoints for pipeline, optimizers, and gradient scalar
|
||||
self.pipeline.load_pipeline(loaded_state["pipeline"])
|
||||
self.pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"])
|
||||
self.optimizers.load_optimizers(loaded_state["optimizers"])
|
||||
self.grad_scaler.load_state_dict(loaded_state["scalers"])
|
||||
CONSOLE.print(f"done loading checkpoint from {load_path}")
|
||||
|
||||
@@ -210,3 +210,11 @@ class Model(nn.Module):
|
||||
"""
|
||||
state = {key.replace("module.", ""): value for key, value in loaded_state["model"].items()}
|
||||
self.load_state_dict(state) # type: ignore
|
||||
|
||||
def update_to_step(self, step: int) -> None:
|
||||
"""Called when loading a model from a checkpoint. Sets any model parameters that change over
|
||||
training to the correct value, based on the training step of the checkpoint.
|
||||
|
||||
Args:
|
||||
step: training step of the loaded checkpoint
|
||||
"""
|
||||
@@ -114,7 +114,8 @@ class TensoRFModel(Model):
|
||||
def reinitialize_optimizer(
|
||||
self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument
|
||||
):
|
||||
resolution = self.upsampling_steps.pop(0)
|
||||
index = self.upsampling_iters.index(step)
|
||||
resolution = self.upsampling_steps[index]
|
||||
|
||||
# upsample the position and direction grids
|
||||
self.field.density_encoding.upsample_grid(resolution)
|
||||
@@ -143,6 +144,19 @@ class TensoRFModel(Model):
|
||||
]
|
||||
return callbacks
|
||||
|
||||
def update_to_step(self, step: int) -> None:
|
||||
if step < self.upsampling_iters[0]:
|
||||
return
|
||||
|
||||
new_iters = list(self.upsampling_iters) + [step + 1]
|
||||
new_iters.sort()
|
||||
|
||||
index = new_iters.index(step + 1)
|
||||
new_grid_resolution = self.upsampling_steps[index - 1]
|
||||
|
||||
self.field.density_encoding.upsample_grid(new_grid_resolution) # type: ignore
|
||||
self.field.color_encoding.upsample_grid(new_grid_resolution) # type: ignore
|
||||
|
||||
def populate_modules(self):
|
||||
"""Set the fields and modules"""
|
||||
super().populate_modules()
|
||||
|
||||
@@ -158,11 +158,12 @@ class Pipeline(nn.Module):
|
||||
def get_average_eval_image_metrics(self, step: Optional[int] = None):
|
||||
"""Iterate over all the images in the eval dataset and get the average."""
|
||||
|
||||
def load_pipeline(self, loaded_state: Dict[str, Any]) -> None:
|
||||
def load_pipeline(self, loaded_state: Dict[str, Any], step: int) -> None:
|
||||
"""Load the checkpoint from the given path
|
||||
|
||||
Args:
|
||||
loaded_state: pre-trained model state dict
|
||||
step: training step of the loaded checkpoint
|
||||
"""
|
||||
|
||||
def get_training_callbacks(
|
||||
@@ -352,13 +353,15 @@ class VanillaPipeline(Pipeline):
|
||||
self.train()
|
||||
return metrics_dict
|
||||
|
||||
def load_pipeline(self, loaded_state: Dict[str, Any]) -> None:
|
||||
def load_pipeline(self, loaded_state: Dict[str, Any], step: int) -> None:
|
||||
"""Load the checkpoint from the given path
|
||||
|
||||
Args:
|
||||
loaded_state: pre-trained model state dict
|
||||
step: training step of the loaded checkpoint
|
||||
"""
|
||||
state = {key.replace("module.", ""): value for key, value in loaded_state.items()}
|
||||
self._model.update_to_step(step)
|
||||
self.load_state_dict(state, strict=False)
|
||||
|
||||
def get_training_callbacks(
|
||||
|
||||
Reference in new issue
Block a user