How to use this VAE on SDXL
trouble-maker007 opened this issue · comments
trouble-maker007 commented
Alfred Increment commented
Any idea?
from diffusers import DiffusionPipeline,StableDiffusionPipeline
import torch
from consistencydecoder import ConsistencyDecoder
from PIL import Image
import numpy as np
pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16, variant="fp16")
pipe = pipe.to("cuda:0")
decoder_consistency = ConsistencyDecoder(device="cuda:0") # Model size: 2.49 GB
prompt = "Self-portrait oil painting, a beautiful cyborg with golden hair, 8k"
latent = pipe(prompt=prompt, output_type="latent")
latent=latent.images[0]
latent=latent.to(torch.float32)/0.13025
latent=latent.unsqueeze(0)
print(latent.size())
with torch.no_grad(), torch.amp.autocast("cuda"):
consistent_latent = decoder_consistency(latent,schedule=[1.0])
image = consistent_latent[0].cpu().numpy()
image = (image + 1.0) * 127.5
image = image.clip(0, 255).astype(np.uint8)
image = Image.fromarray(image.transpose(1, 2, 0))
image.save("con.png")
bonlime commented
this VAE would not work for SDXL, since it has a different VAE encoder