mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-18 22:04:02 +08:00
adding dream embedding
This commit is contained in:
1 parent
7d8c6a98fb
commit
d049e901d1
6 files changed
+223
-185
No files matched your search
@@ -188,6 +188,42 @@ class StableDiffusion(nn.Module):
|
||||
|
||||
return text_embeddings
|
||||
|
||||
def sds_loss_latent(
|
||||
self,
|
||||
text_embeddings: TensorType["N", "max_length", "embed_dim"],
|
||||
latents: TensorType["BS", 3, "H", "W"],
|
||||
guidance_scale: float = 100.0,
|
||||
grad_scaler: Optional[GradScaler] = None,
|
||||
) -> torch.Tensor:
|
||||
# Disable autocast when using diffusers pipeline.
|
||||
with torch.autocast(device_type="cuda", enabled=False):
|
||||
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
|
||||
latents = latents.half()
|
||||
# predict the noise residual with unet, NO grad!
|
||||
with torch.no_grad():
|
||||
# add noise
|
||||
noise = torch.randn_like(latents)
|
||||
latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore
|
||||
# pred noise
|
||||
latent_model_input = torch.cat([latents_noisy] * 2)
|
||||
noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
|
||||
|
||||
# perform guidance
|
||||
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
|
||||
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
|
||||
|
||||
# w(t), sigma_t^2
|
||||
w = 1 - self.alphas[t]
|
||||
|
||||
grad = w * (noise_pred - noise)
|
||||
grad = torch.nan_to_num(grad)
|
||||
|
||||
if grad_scaler is not None:
|
||||
latents = grad_scaler.scale(latents)
|
||||
loss = _SDSGradient.apply(latents, grad)
|
||||
|
||||
return loss
|
||||
|
||||
def sds_loss(
|
||||
self,
|
||||
text_embeddings: TensorType["N", "max_length", "embed_dim"],
|
||||
@@ -226,7 +262,7 @@ class StableDiffusion(nn.Module):
|
||||
# w(t), sigma_t^2
|
||||
w = 1 - self.alphas[t]
|
||||
|
||||
grad = w * (noise_pred - noise)
|
||||
grad = w * (noise_pred - noise) * 0.01
|
||||
grad = torch.nan_to_num(grad)
|
||||
|
||||
if grad_scaler is not None:
|
||||
|
||||
Reference in new issue
Block a user