mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-11 10:34:29 +08:00
f67dca6f5b
* refactor code to move graph/dataset setup to respective files * consolidated eval and benchmark script * moved setup optimizers to its own thing
129 lines
4.6 KiB
Python
129 lines
4.6 KiB
Python
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""
|
|
Optimizers class.
|
|
"""
|
|
|
|
from typing import Any, Dict, List
|
|
|
|
import numpy as np
|
|
from hydra.utils import instantiate
|
|
from omegaconf import DictConfig
|
|
from torch.nn import Parameter
|
|
from torch.optim.lr_scheduler import LambdaLR
|
|
|
|
from pyrad.utils import writer
|
|
|
|
|
|
def setup_optimizers(config: DictConfig, param_groups: Dict[str, List[Parameter]]) -> "Optimizers":
|
|
"""Helper to set up the optimizers
|
|
|
|
Args:
|
|
config (DictConfig): _description_
|
|
param_groups (Dict[str, List[Parameter]]): _description_
|
|
|
|
Returns:
|
|
Optimizers: _description_
|
|
"""
|
|
return Optimizers(config, param_groups)
|
|
|
|
|
|
class ExponentialDecaySchedule(LambdaLR):
|
|
"""Exponential learning rate decay function.
|
|
See https://github.com/google-research/google-research/blob/
|
|
fd2cea8cdd86b3ed2c640cbe5561707639e682f3/jaxnerf/nerf/utils.py#L360
|
|
for details.
|
|
"""
|
|
|
|
def __init__(self, optimizer, lr_init, lr_final, max_steps, lr_delay_steps=0, lr_delay_mult=1.0) -> None:
|
|
def func(step):
|
|
if lr_delay_steps > 0:
|
|
delay_rate = lr_delay_mult + (1 - lr_delay_mult) * np.sin(
|
|
0.5 * np.pi * np.clip(step / lr_delay_steps, 0, 1)
|
|
)
|
|
else:
|
|
delay_rate = 1.0
|
|
t = np.clip(step / max_steps, 0, 1)
|
|
log_lerp = np.exp(np.log(lr_init) * (1 - t) + np.log(lr_final) * t)
|
|
multiplier = (
|
|
log_lerp / lr_init
|
|
) # divided by lr_init because the multiplier is with the initial learning rate
|
|
return delay_rate * multiplier
|
|
|
|
super().__init__(optimizer, lr_lambda=func)
|
|
|
|
|
|
class Optimizers:
|
|
"""_summary_"""
|
|
|
|
def __init__(self, config: DictConfig, param_groups: Dict[str, List[Parameter]]):
|
|
"""_summary_
|
|
|
|
Args:
|
|
config (DictConfig): _description_
|
|
param_dict (Dict[str, List[Parameter]]): _description_
|
|
"""
|
|
self.config = config
|
|
self.optimizers = {}
|
|
self.schedulers = {}
|
|
for param_group_name, params in param_groups.items():
|
|
lr_init = config[param_group_name].optimizer.lr
|
|
self.optimizers[param_group_name] = instantiate(config[param_group_name].optimizer, params=params)
|
|
if config[param_group_name].scheduler:
|
|
self.schedulers[param_group_name] = instantiate(
|
|
config[param_group_name].scheduler, optimizer=self.optimizers[param_group_name], lr_init=lr_init
|
|
)
|
|
|
|
def optimizer_step(self, param_group_name: str) -> None:
|
|
"""fetch and step corresponding optimizer
|
|
|
|
Args:
|
|
param_group_name (str): name of optimizer to step forward
|
|
"""
|
|
self.optimizers[param_group_name].step()
|
|
|
|
def scheduler_step(self, param_group_name: str) -> None:
|
|
"""fetch and step corresponding scheduler
|
|
|
|
Args:
|
|
param_group_name (str): name of scheduler to step forward
|
|
"""
|
|
if self.config.param_group_name.scheduler:
|
|
self.schedulers[param_group_name].step()
|
|
|
|
def zero_grad_all(self):
|
|
"""zero the gradients for all optimizer parameters"""
|
|
for _, optimizer in self.optimizers.items():
|
|
optimizer.zero_grad()
|
|
|
|
def optimizer_step_all(self):
|
|
"""Run step for all optimizers."""
|
|
for _, optimizer in self.optimizers.items():
|
|
# note that they key is the parameter name
|
|
optimizer.step()
|
|
|
|
def scheduler_step_all(self, step):
|
|
"""Run step for all schedulers."""
|
|
for param_group_name, scheduler in self.schedulers.items():
|
|
scheduler.step()
|
|
# TODO(ethan): clean this up. why is there indexing into a list?
|
|
lr = scheduler.get_last_lr()[0]
|
|
writer.put_scalar(name=f"learning_rate/{param_group_name}", scalar=lr, step=step)
|
|
|
|
def load_optimizers(self, loaded_state: Dict[str, Any]) -> None:
|
|
"""Helper to load the optimizer state from previous checkpoint"""
|
|
for k, v in loaded_state["optimizers"].items():
|
|
self.optimizers[k].load_state_dict(v)
|