adding dream embedding

This commit is contained in:
terrancewang committed 2023-03-02 13:55:33 -08:00
1 parent 7d8c6a98fb
commit d049e901d1
6 files changed
+223 -185

No files matched your search

+37 -1
View File
@@ -188,6 +188,42 @@ class StableDiffusion(nn.Module):
return text_embeddings
def sds_loss_latent(
self,
text_embeddings: TensorType["N", "max_length", "embed_dim"],
latents: TensorType["BS", 3, "H", "W"],
guidance_scale: float = 100.0,
grad_scaler: Optional[GradScaler] = None,
) -> torch.Tensor:
# Disable autocast when using diffusers pipeline.
with torch.autocast(device_type="cuda", enabled=False):
t = torch.randint(self.min_step, self.max_step + 1, [1], dtype=torch.long, device=self.device)
latents = latents.half()
# predict the noise residual with unet, NO grad!
with torch.no_grad():
# add noise
noise = torch.randn_like(latents)
latents_noisy = self.scheduler.add_noise(latents, noise, t) # type: ignore
# pred noise
latent_model_input = torch.cat([latents_noisy] * 2)
noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
# perform guidance
noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
noise_pred = noise_pred_text + guidance_scale * (noise_pred_text - noise_pred_uncond)
# w(t), sigma_t^2
w = 1 - self.alphas[t]
grad = w * (noise_pred - noise)
grad = torch.nan_to_num(grad)
if grad_scaler is not None:
latents = grad_scaler.scale(latents)
loss = _SDSGradient.apply(latents, grad)
return loss
def sds_loss(
self,
text_embeddings: TensorType["N", "max_length", "embed_dim"],
@@ -226,7 +262,7 @@ class StableDiffusion(nn.Module):
# w(t), sigma_t^2
w = 1 - self.alphas[t]
grad = w * (noise_pred - noise)
grad = w * (noise_pred - noise) * 0.01
grad = torch.nan_to_num(grad)
if grad_scaler is not None: