fix pytorch 2.0 optimizer no inf check error (#1919)

* fix pytorch 2.0 optimizer no inf check error

* Update optimizers.py

nerfstudio/engine/optimizers.py

@@ -127,7 +127,8 @@ class Optimizers:
             if max_norm is not None:
                 grad_scaler.unscale_(optimizer)
                 torch.nn.utils.clip_grad_norm_(self.parameters[param_group], max_norm)
-            grad_scaler.step(optimizer)
+            if any(any(p.grad is not None for p in g["params"]) for g in optimizer.param_groups):
+                grad_scaler.step(optimizer)
 
     def optimizer_step_all(self) -> None:
         """Run step for all optimizers."""