Fix tensorf chkpt loading (#1266)

* setting tensorf default from wandb to viewer

* adding functionality to adjust model depending on the step of the checkpoint that is loaded

* cleaning code

* bug fix

* fixing style
This commit is contained in:
Terrance Wang authored and GitHub committed 2023-01-20 12:00:51 -08:00
1 parent 9eb880aa91
commit 73bffe058f
4 files changed
+29 -4

No files matched your search

+1 -1
View File
@@ -315,7 +315,7 @@ class Trainer:
loaded_state = torch.load(load_path, map_location="cpu")
self._start_step = loaded_state["step"] + 1
# load the checkpoints for pipeline, optimizers, and gradient scalar
self.pipeline.load_pipeline(loaded_state["pipeline"])
self.pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"])
self.optimizers.load_optimizers(loaded_state["optimizers"])
self.grad_scaler.load_state_dict(loaded_state["scalers"])
CONSOLE.print(f"done loading checkpoint from {load_path}")
+8
View File
@@ -210,3 +210,11 @@ class Model(nn.Module):
"""
state = {key.replace("module.", ""): value for key, value in loaded_state["model"].items()}
self.load_state_dict(state) # type: ignore
def update_to_step(self, step: int) -> None:
"""Called when loading a model from a checkpoint. Sets any model parameters that change over
training to the correct value, based on the training step of the checkpoint.
Args:
step: training step of the loaded checkpoint
"""
+15 -1
View File
@@ -114,7 +114,8 @@ class TensoRFModel(Model):
def reinitialize_optimizer(
self, training_callback_attributes: TrainingCallbackAttributes, step: int # pylint: disable=unused-argument
):
resolution = self.upsampling_steps.pop(0)
index = self.upsampling_iters.index(step)
resolution = self.upsampling_steps[index]
# upsample the position and direction grids
self.field.density_encoding.upsample_grid(resolution)
@@ -143,6 +144,19 @@ class TensoRFModel(Model):
]
return callbacks
def update_to_step(self, step: int) -> None:
if step < self.upsampling_iters[0]:
return
new_iters = list(self.upsampling_iters) + [step + 1]
new_iters.sort()
index = new_iters.index(step + 1)
new_grid_resolution = self.upsampling_steps[index - 1]
self.field.density_encoding.upsample_grid(new_grid_resolution) # type: ignore
self.field.color_encoding.upsample_grid(new_grid_resolution) # type: ignore
def populate_modules(self):
"""Set the fields and modules"""
super().populate_modules()
+5 -2
View File
@@ -158,11 +158,12 @@ class Pipeline(nn.Module):
def get_average_eval_image_metrics(self, step: Optional[int] = None):
"""Iterate over all the images in the eval dataset and get the average."""
def load_pipeline(self, loaded_state: Dict[str, Any]) -> None:
def load_pipeline(self, loaded_state: Dict[str, Any], step: int) -> None:
"""Load the checkpoint from the given path
Args:
loaded_state: pre-trained model state dict
step: training step of the loaded checkpoint
"""
def get_training_callbacks(
@@ -352,13 +353,15 @@ class VanillaPipeline(Pipeline):
self.train()
return metrics_dict
def load_pipeline(self, loaded_state: Dict[str, Any]) -> None:
def load_pipeline(self, loaded_state: Dict[str, Any], step: int) -> None:
"""Load the checkpoint from the given path
Args:
loaded_state: pre-trained model state dict
step: training step of the loaded checkpoint
"""
state = {key.replace("module.", ""): value for key, value in loaded_state.items()}
self._model.update_to_step(step)
self.load_state_dict(state, strict=False)
def get_training_callbacks(