Add saving/loading scheduler state_dict (#2203)

- Add saving/loading scheduler stat_dict
This commit is contained in:
hyperzy authored and GitHub committed 2023-07-12 14:14:59 +00:00
1 parent 49a4be635d
commit cebe787bed
2 files changed
+11

No files matched your search

+9
View File
@@ -159,3 +159,12 @@ class Optimizers:
"""
for k, v in loaded_state.items():
self.optimizers[k].load_state_dict(v)
def load_schedulers(self, loaded_state: Dict[str, Any]) -> None:
"""Helper to load the scheduler state from previous checkpoint
Args:
loaded_state: the state from the previous checkpoint
"""
for k, v in loaded_state.items():
self.schedulers[k].load_state_dict(v)
+2
View File
@@ -406,6 +406,7 @@ class Trainer:
# load the checkpoints for pipeline, optimizers, and gradient scalar
self.pipeline.load_pipeline(loaded_state["pipeline"], loaded_state["step"])
self.optimizers.load_optimizers(loaded_state["optimizers"])
self.optimizers.load_schedulers(loaded_state["schedulers"])
self.grad_scaler.load_state_dict(loaded_state["scalers"])
CONSOLE.print(f"Done loading Nerfstudio checkpoint from {load_path}")
elif load_checkpoint is not None:
@@ -439,6 +440,7 @@ class Trainer:
if hasattr(self.pipeline, "module")
else self.pipeline.state_dict(),
"optimizers": {k: v.state_dict() for (k, v) in self.optimizers.optimizers.items()},
"schedulers": {k: v.state_dict() for (k, v) in self.optimizers.schedulers.items()},
"scalers": self.grad_scaler.state_dict(),
},
ckpt_path,