mirror of
https://github.com/nerfstudio-project/nerfstudio.git
synced 2026-11-19 14:23:57 +08:00
Start SD cleanup
This commit is contained in:
1 parent
bd5ff4868e
commit
a45c6ca962
2 files changed
+102
-46
No files matched your search
@@ -14,19 +14,37 @@
|
|||||||
|
|
||||||
# From https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py
|
# From https://github.com/ashawkey/stable-dreamfusion/blob/main/nerf/sd.py
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Optional, Union
|
||||||
|
|
||||||
import matplotlib.pyplot as plt
|
import matplotlib.pyplot as plt
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
import torch.nn.functional as F
|
import torch.nn.functional as F
|
||||||
from diffusers import PNDMScheduler, StableDiffusionPipeline
|
import tyro
|
||||||
from rich.console import Console
|
from rich.console import Console
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from transformers import logging
|
from torchtyping import TensorType
|
||||||
|
|
||||||
CONSOLE = Console(width=120)
|
CONSOLE = Console(width=120)
|
||||||
|
|
||||||
|
try:
|
||||||
|
from diffusers import PNDMScheduler, StableDiffusionPipeline
|
||||||
|
from transformers import logging
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
CONSOLE.print("[bold red]Missing Stable Diffusion packages.")
|
||||||
|
CONSOLE.print(r"Install using [yellow]pip install nerfstudio\[gen][/yellow]")
|
||||||
|
CONSOLE.print(r"or [yellow]pip install -e .\[gen][/yellow] if installing from source.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
logging.set_verbosity_error()
|
logging.set_verbosity_error()
|
||||||
IMG_DIM = 512
|
IMG_DIM = 512
|
||||||
const_scale = 0.18215
|
CONST_SCALE = 0.18215
|
||||||
|
|
||||||
|
SD_SOURCE = "runwayml/stable-diffusion-v1-5"
|
||||||
|
CLIP_SOURCE = "openai/clip-vit-large-patch14"
|
||||||
|
|
||||||
|
|
||||||
class StableDiffusion(nn.Module):
|
class StableDiffusion(nn.Module):
|
||||||
@@ -37,18 +55,6 @@ class StableDiffusion(nn.Module):
|
|||||||
|
|
||||||
self.device = device
|
self.device = device
|
||||||
|
|
||||||
try:
|
|
||||||
with open("./HF_TOKEN", "r") as f:
|
|
||||||
self.token = f.read().replace("\n", "")
|
|
||||||
CONSOLE.print(f"Hugging face access token loaded.")
|
|
||||||
except FileNotFoundError as e:
|
|
||||||
raise AssertionError(
|
|
||||||
f"Cannot read hugging face token, make sure to save your Hugging Face token at `./HF_TOKEN`"
|
|
||||||
)
|
|
||||||
|
|
||||||
SD_SOURCE = "stabilityai/stable-diffusion-2-1"
|
|
||||||
CLIP_SOURCE = "openai/clip-vit-large-patch14"
|
|
||||||
|
|
||||||
self.num_train_timesteps = 1000
|
self.num_train_timesteps = 1000
|
||||||
self.min_step = int(self.num_train_timesteps * 0.02)
|
self.min_step = int(self.num_train_timesteps * 0.02)
|
||||||
self.max_step = int(self.num_train_timesteps * 0.98)
|
self.max_step = int(self.num_train_timesteps * 0.98)
|
||||||
@@ -62,6 +68,7 @@ class StableDiffusion(nn.Module):
|
|||||||
self.alphas = self.scheduler.alphas_cumprod.to(self.device)
|
self.alphas = self.scheduler.alphas_cumprod.to(self.device)
|
||||||
|
|
||||||
pipe = StableDiffusionPipeline.from_pretrained(SD_SOURCE, torch_dtype=torch.float16)
|
pipe = StableDiffusionPipeline.from_pretrained(SD_SOURCE, torch_dtype=torch.float16)
|
||||||
|
assert pipe is not None
|
||||||
pipe = pipe.to(self.device)
|
pipe = pipe.to(self.device)
|
||||||
|
|
||||||
# MEMORY IMPROVEMENTS
|
# MEMORY IMPROVEMENTS
|
||||||
@@ -80,10 +87,20 @@ class StableDiffusion(nn.Module):
|
|||||||
|
|
||||||
self.unet.to(memory_format=torch.channels_last)
|
self.unet.to(memory_format=torch.channels_last)
|
||||||
|
|
||||||
CONSOLE.print(f"Stable Diffusion loaded!")
|
CONSOLE.print("Stable Diffusion loaded!")
|
||||||
|
|
||||||
def get_text_embeds(self, prompt, negative_prompt):
|
def get_text_embeds(
|
||||||
# prompt, negative_prompt: [str]
|
self, prompt: Union[str, List[str]], negative_prompt: Union[str, List[str]]
|
||||||
|
) -> TensorType[2, "max_length", "embed_dim"]:
|
||||||
|
"""Get text embeddings for prompt and negative prompt
|
||||||
|
|
||||||
|
Args:
|
||||||
|
prompt: Prompt text
|
||||||
|
negative_prompt: Negative prompt text
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Text embeddings
|
||||||
|
"""
|
||||||
|
|
||||||
# Tokenize text and get embeddings
|
# Tokenize text and get embeddings
|
||||||
text_input = self.tokenizer(
|
text_input = self.tokenizer(
|
||||||
@@ -107,6 +124,7 @@ class StableDiffusion(nn.Module):
|
|||||||
|
|
||||||
# Cat for final embeddings
|
# Cat for final embeddings
|
||||||
text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
|
text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
|
||||||
|
|
||||||
return text_embeddings
|
return text_embeddings
|
||||||
|
|
||||||
def sds_loss(self, text_embeddings, nerf_output, guidance_scale=100):
|
def sds_loss(self, text_embeddings, nerf_output, guidance_scale=100):
|
||||||
@@ -186,7 +204,7 @@ class StableDiffusion(nn.Module):
|
|||||||
|
|
||||||
def latents_to_img(self, latents):
|
def latents_to_img(self, latents):
|
||||||
|
|
||||||
latents = 1 / const_scale * latents
|
latents = 1 / CONST_SCALE * latents
|
||||||
|
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
imgs = self.auto_encoder.decode(latents).sample
|
imgs = self.auto_encoder.decode(latents).sample
|
||||||
@@ -200,11 +218,30 @@ class StableDiffusion(nn.Module):
|
|||||||
imgs = 2 * imgs - 1
|
imgs = 2 * imgs - 1
|
||||||
|
|
||||||
posterior = self.auto_encoder.encode(imgs).latent_dist
|
posterior = self.auto_encoder.encode(imgs).latent_dist
|
||||||
latents = posterior.sample() * const_scale
|
latents = posterior.sample() * CONST_SCALE
|
||||||
|
|
||||||
return latents
|
return latents
|
||||||
|
|
||||||
def prompt_to_img(self, prompts, negative_prompts="", num_inference_steps=50, guidance_scale=7.5, latents=None):
|
def prompt_to_img(
|
||||||
|
self,
|
||||||
|
prompts: Union[str, List[str]],
|
||||||
|
negative_prompts: Union[str, List[str]] = "",
|
||||||
|
num_inference_steps: int = 50,
|
||||||
|
guidance_scale: float = 7.5,
|
||||||
|
latents=None,
|
||||||
|
) -> np.ndarray:
|
||||||
|
"""Generate an images from a prompts.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
prompts: The prompt to generate an image from.
|
||||||
|
negative_prompts: The negative prompt to generate an image from.
|
||||||
|
num_inference_steps: The number of inference steps to perform.
|
||||||
|
guidance_scale: The scale of the guidance.
|
||||||
|
latents: The latents to start from, defaults to random.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The generated image.
|
||||||
|
"""
|
||||||
|
|
||||||
prompts = [prompts] if isinstance(prompts, str) else prompts
|
prompts = [prompts] if isinstance(prompts, str) else prompts
|
||||||
negative_prompts = [negative_prompts] if isinstance(negative_prompts, str) else negative_prompts
|
negative_prompts = [negative_prompts] if isinstance(negative_prompts, str) else negative_prompts
|
||||||
@@ -218,37 +255,50 @@ class StableDiffusion(nn.Module):
|
|||||||
guidance_scale=guidance_scale,
|
guidance_scale=guidance_scale,
|
||||||
) # [1, 4, resolution, resolution]
|
) # [1, 4, resolution, resolution]
|
||||||
|
|
||||||
diffused_img = self.latents_to_img(latents)
|
diffused_img = self.latents_to_img(latents.half())
|
||||||
diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy()
|
diffused_img = diffused_img.detach().cpu().permute(0, 2, 3, 1).numpy()
|
||||||
diffused_img = (diffused_img * 255).round().astype("uint8")
|
diffused_img = (diffused_img * 255).round().astype("uint8")
|
||||||
|
|
||||||
return diffused_img
|
return diffused_img
|
||||||
|
|
||||||
|
def forward(
|
||||||
|
self, prompts, negative_prompts="", num_inference_steps=50, guidance_scale=7.5, latents=None
|
||||||
|
) -> np.ndarray:
|
||||||
|
"""Generate an image from a prompt.
|
||||||
|
|
||||||
def seed_everything(seed):
|
Args:
|
||||||
|
prompts: The prompt to generate an image from.
|
||||||
|
negative_prompts: The negative prompt to generate an image from.
|
||||||
|
num_inference_steps: The number of inference steps to perform.
|
||||||
|
guidance_scale: The scale of the guidance.
|
||||||
|
latents: The latents to start from, defaults to random.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The generated image.
|
||||||
|
"""
|
||||||
|
return self.prompt_to_img(prompts, negative_prompts, num_inference_steps, guidance_scale, latents)
|
||||||
|
|
||||||
|
|
||||||
|
def generate_image(
|
||||||
|
prompt: str, negative: str = "", seed: int = 0, steps: int = 50, save_path: Path = Path("test_sd.png")
|
||||||
|
):
|
||||||
|
"""Generate an image from a prompt using Stable Diffusion.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
prompt: The prompt to use.
|
||||||
|
negative: The negative prompt to use.
|
||||||
|
seed: The random seed to use.
|
||||||
|
steps: The number of steps to use.
|
||||||
|
save_path: The path to save the image to.
|
||||||
|
"""
|
||||||
torch.manual_seed(seed)
|
torch.manual_seed(seed)
|
||||||
torch.cuda.manual_seed(seed)
|
torch.cuda.manual_seed(seed)
|
||||||
|
cuda_device = torch.device("cuda")
|
||||||
|
with torch.no_grad():
|
||||||
|
sd = StableDiffusion(cuda_device)
|
||||||
|
imgs = sd.prompt_to_img(prompt, negative, steps)
|
||||||
|
plt.imsave(str(save_path), imgs[0])
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
tyro.cli(generate_image)
|
||||||
import argparse
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("prompt", type=str)
|
|
||||||
parser.add_argument("--negative", default="", type=str)
|
|
||||||
parser.add_argument("--seed", type=int, default=0)
|
|
||||||
parser.add_argument("--steps", type=int, default=50)
|
|
||||||
opt = parser.parse_args()
|
|
||||||
|
|
||||||
seed_everything(opt.seed)
|
|
||||||
|
|
||||||
device = torch.device("cuda")
|
|
||||||
|
|
||||||
with torch.no_grad():
|
|
||||||
|
|
||||||
sd = StableDiffusion(device)
|
|
||||||
|
|
||||||
imgs = sd.prompt_to_img(opt.prompt, opt.negative, opt.steps)
|
|
||||||
|
|
||||||
plt.imsave("test_sd.png", imgs[0])
|
|
||||||
+8
-2
@@ -45,8 +45,6 @@ dependencies = [
|
|||||||
"scikit-image>=0.19.3",
|
"scikit-image>=0.19.3",
|
||||||
"tensorboard==2.9.0",
|
"tensorboard==2.9.0",
|
||||||
"torch==1.12.1",
|
"torch==1.12.1",
|
||||||
"diffusers==0.9.0",
|
|
||||||
"transformers==4.24.0",
|
|
||||||
"torchmetrics[image]>=0.9.3",
|
"torchmetrics[image]>=0.9.3",
|
||||||
"torchtyping>=0.1.4",
|
"torchtyping>=0.1.4",
|
||||||
"torchvision>=0.13.0",
|
"torchvision>=0.13.0",
|
||||||
@@ -61,6 +59,14 @@ dependencies = [
|
|||||||
|
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
|
|
||||||
|
# Generative related dependencies
|
||||||
|
gen = [
|
||||||
|
"diffusers==0.9.0",
|
||||||
|
"transformers==4.24.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
# Development packages
|
# Development packages
|
||||||
dev = [
|
dev = [
|
||||||
"black[jupyter]==22.3.0",
|
"black[jupyter]==22.3.0",
|
||||||
|
|||||||
Reference in new issue
Block a user