mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-20 14:45:38 +08:00
1 parent
62f492f1bc
commit
e13d401b16
97 files changed
+235712
-235721
No files matched your search
@@ -0,0 +1,152 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""
|
||||
Optimizers class.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Type
|
||||
|
||||
import torch
|
||||
from torch.cuda.amp.grad_scaler import GradScaler
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from nerfstudio.configs.base_config import PrintableConfig
|
||||
from nerfstudio.utils import writer
|
||||
|
||||
|
||||
# Optimizer related configs
|
||||
@dataclass
|
||||
class OptimizerConfig(PrintableConfig):
|
||||
"""Basic optimizer config with RAdam"""
|
||||
|
||||
_target: Type = torch.optim.Adam
|
||||
lr: float = 0.0005
|
||||
eps: float = 1e-08
|
||||
|
||||
# TODO: somehow make this more generic. i dont like the idea of overriding the setup function
|
||||
# but also not sure how to go about passing things into predefined torch objects.
|
||||
def setup(self, params) -> Any:
|
||||
"""Returns the instantiated object using the config."""
|
||||
kwargs = vars(self)
|
||||
del kwargs["_target"]
|
||||
return self._target(params, **kwargs)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AdamOptimizerConfig(OptimizerConfig):
|
||||
"""Basic optimizer config with Adam"""
|
||||
|
||||
_target: Type = torch.optim.Adam
|
||||
|
||||
|
||||
@dataclass
|
||||
class RAdamOptimizerConfig(OptimizerConfig):
|
||||
"""Basic optimizer config with RAdam"""
|
||||
|
||||
_target: Type = torch.optim.RAdam
|
||||
|
||||
|
||||
def setup_optimizers(config: Dict[str, Any], param_groups: Dict[str, List[Parameter]]) -> "Optimizers":
|
||||
"""Helper to set up the optimizers
|
||||
|
||||
Args:
|
||||
config: The optimizer configuration object.
|
||||
param_groups: A dictionary of parameter groups to optimize.
|
||||
|
||||
Returns:
|
||||
The optimizers object.
|
||||
"""
|
||||
return Optimizers(config, param_groups)
|
||||
|
||||
|
||||
class Optimizers:
|
||||
"""A set of optimizers.
|
||||
|
||||
Args:
|
||||
config: The optimizer configuration object.
|
||||
param_groups: A dictionary of parameter groups to optimize.
|
||||
"""
|
||||
|
||||
def __init__(self, config: Dict[str, Any], param_groups: Dict[str, List[Parameter]]):
|
||||
self.config = config
|
||||
self.optimizers = {}
|
||||
self.schedulers = {}
|
||||
for param_group_name, params in param_groups.items():
|
||||
lr_init = config[param_group_name]["optimizer"].lr
|
||||
self.optimizers[param_group_name] = config[param_group_name]["optimizer"].setup(params=params)
|
||||
if config[param_group_name]["scheduler"]:
|
||||
self.schedulers[param_group_name] = config[param_group_name]["scheduler"].setup(
|
||||
optimizer=self.optimizers[param_group_name], lr_init=lr_init
|
||||
)
|
||||
|
||||
def optimizer_step(self, param_group_name: str) -> None:
|
||||
"""Fetch and step corresponding optimizer.
|
||||
|
||||
Args:
|
||||
param_group_name: name of optimizer to step forward
|
||||
"""
|
||||
self.optimizers[param_group_name].step()
|
||||
|
||||
def scheduler_step(self, param_group_name: str) -> None:
|
||||
"""Fetch and step corresponding scheduler.
|
||||
|
||||
Args:
|
||||
param_group_name: name of scheduler to step forward
|
||||
"""
|
||||
if self.config.param_group_name.scheduler: # type: ignore
|
||||
self.schedulers[param_group_name].step()
|
||||
|
||||
def zero_grad_all(self) -> None:
|
||||
"""Zero the gradients for all optimizer parameters."""
|
||||
for _, optimizer in self.optimizers.items():
|
||||
optimizer.zero_grad()
|
||||
|
||||
def optimizer_scaler_step_all(self, grad_scaler: GradScaler) -> None:
|
||||
"""Take an optimizer step using a grad scaler.
|
||||
|
||||
Args:
|
||||
grad_scaler: GradScaler to use
|
||||
"""
|
||||
for _, optimizer in self.optimizers.items():
|
||||
grad_scaler.step(optimizer)
|
||||
|
||||
def optimizer_step_all(self):
|
||||
"""Run step for all optimizers."""
|
||||
for _, optimizer in self.optimizers.items():
|
||||
# note that they key is the parameter name
|
||||
optimizer.step()
|
||||
|
||||
def scheduler_step_all(self, step: int) -> None:
|
||||
"""Run step for all schedulers.
|
||||
|
||||
Args:
|
||||
step: the current step
|
||||
"""
|
||||
for param_group_name, scheduler in self.schedulers.items():
|
||||
scheduler.step()
|
||||
# TODO(ethan): clean this up. why is there indexing into a list?
|
||||
lr = scheduler.get_last_lr()[0]
|
||||
writer.put_scalar(name=f"learning_rate/{param_group_name}", scalar=lr, step=step)
|
||||
|
||||
def load_optimizers(self, loaded_state: Dict[str, Any]) -> None:
|
||||
"""Helper to load the optimizer state from previous checkpoint
|
||||
|
||||
Args:
|
||||
loaded_state: the state from the previous checkpoint
|
||||
"""
|
||||
for k, v in loaded_state.items():
|
||||
self.optimizers[k].load_state_dict(v)
|
||||
@@ -0,0 +1,73 @@
|
||||
# Copyright 2022 The Plenoptix Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Scheduler Classes"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Type
|
||||
|
||||
import numpy as np
|
||||
from torch.optim.lr_scheduler import LambdaLR
|
||||
|
||||
from nerfstudio.configs.base_config import InstantiateConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchedulerConfig(InstantiateConfig):
|
||||
"""Basic scheduler config with self-defined exponential decay schedule"""
|
||||
|
||||
_target: Type = field(default_factory=lambda: ExponentialDecaySchedule)
|
||||
lr_final: float = 0.000005
|
||||
max_steps: int = 1000000
|
||||
|
||||
# TODO: somehow make this more generic. i dont like the idea of overriding the setup function
|
||||
# but also not sure how to go about passing things into predefined torch objects.
|
||||
def setup(self, optimizer=None, lr_init=None, **kwargs) -> Any:
|
||||
"""Returns the instantiated object using the config."""
|
||||
return self._target(optimizer, lr_init, self.lr_final, self.max_steps)
|
||||
|
||||
|
||||
class ExponentialDecaySchedule(LambdaLR):
|
||||
"""Exponential learning rate decay function.
|
||||
See https://github.com/google-research/google-research/blob/
|
||||
fd2cea8cdd86b3ed2c640cbe5561707639e682f3/jaxnerf/nerf/utils.py#L360
|
||||
for details.
|
||||
|
||||
Args:
|
||||
optimizer: The optimizer to update.
|
||||
lr_init: The initial learning rate.
|
||||
lr_final: The final learning rate.
|
||||
max_steps: The maximum number of steps.
|
||||
lr_delay_steps: The number of steps to delay the learning rate.
|
||||
lr_delay_mult: The multiplier for the learning rate after the delay.
|
||||
"""
|
||||
|
||||
config: SchedulerConfig
|
||||
|
||||
def __init__(self, optimizer, lr_init, lr_final, max_steps, lr_delay_steps=0, lr_delay_mult=1.0) -> None:
|
||||
def func(step):
|
||||
if lr_delay_steps > 0:
|
||||
delay_rate = lr_delay_mult + (1 - lr_delay_mult) * np.sin(
|
||||
0.5 * np.pi * np.clip(step / lr_delay_steps, 0, 1)
|
||||
)
|
||||
else:
|
||||
delay_rate = 1.0
|
||||
t = np.clip(step / max_steps, 0, 1)
|
||||
log_lerp = np.exp(np.log(lr_init) * (1 - t) + np.log(lr_final) * t)
|
||||
multiplier = (
|
||||
log_lerp / lr_init
|
||||
) # divided by lr_init because the multiplier is with the initial learning rate
|
||||
return delay_rate * multiplier
|
||||
|
||||
super().__init__(optimizer, lr_lambda=func)
|
||||
@@ -28,8 +28,8 @@ import torch
|
||||
from rich import console
|
||||
from torch.cuda.amp.grad_scaler import GradScaler
|
||||
|
||||
from nerfstudio.configs import base as cfg
|
||||
from nerfstudio.optimizers.optimizers import Optimizers, setup_optimizers
|
||||
from nerfstudio.configs import base_config as cfg
|
||||
from nerfstudio.engine.optimizers import Optimizers, setup_optimizers
|
||||
from nerfstudio.pipelines.base import VanillaPipeline
|
||||
from nerfstudio.utils import profiler, writer
|
||||
from nerfstudio.utils.callbacks import (
|
||||
|
||||
Reference in new issue
Block a user