diff --git a/nerfstudio/engine/optimizers.py b/nerfstudio/engine/optimizers.py
index 3f01fb2e9..3ff693685 100644
--- a/nerfstudio/engine/optimizers.py
+++ b/nerfstudio/engine/optimizers.py
@@ -127,7 +127,8 @@ class Optimizers:
             if max_norm is not None:
                 grad_scaler.unscale_(optimizer)
                 torch.nn.utils.clip_grad_norm_(self.parameters[param_group], max_norm)
-            grad_scaler.step(optimizer)
+            if any(any(p.grad is not None for p in g["params"]) for g in optimizer.param_groups):
+                grad_scaler.step(optimizer)
 
     def optimizer_step_all(self) -> None:
         """Run step for all optimizers."""