Fix exponential decay warmup (#2002)

Fix exponendial decay warmup
This commit is contained in:
Matthew Tancik authored and GitHub committed 2023-05-25 10:07:22 -07:00
1 parent a8bce87ada
commit 400ab5a313
1 file changed
+1 -1
+1 -1
View File
@@ -122,7 +122,7 @@ class ExponentialDecayScheduler(Scheduler):
def func(step):
if step < self.config.warmup_steps:
if self.config.ramp == "cosine":
lr = self.config.lr_pre_warmup + (1 - self.config.lr_pre_warmup) * np.sin(
lr = self.config.lr_pre_warmup + (lr_init - self.config.lr_pre_warmup) * np.sin(
0.5 * np.pi * np.clip(step / self.config.warmup_steps, 0, 1)
)
else: