methods.py

import PIL
from PIL import Image
import numpy as np
import torch
import cv2 as cv
import random
import os
import spaces
import gradio as gr

from diffusers import DiffusionPipeline
from peft import PeftModel, LoraConfig

from diffusers import (
    StableDiffusionPipeline,
    StableDiffusionControlNetPipeline,
    StableDiffusionControlNetImg2ImgPipeline,
    DPMSolverMultistepScheduler,
    PNDMScheduler,
    ControlNetModel
)
from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback
from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion import rescale_noise_cfg, retrieve_timesteps
from diffusers.pipelines.stable_diffusion import StableDiffusionPipelineOutput
from diffusers.utils.torch_utils import randn_tensor
from diffusers.utils import load_image, make_image_grid


MAX_SEED = np.iinfo(np.int32).max
MAX_IMAGE_SIZE = 1024
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"

model_id_default = "sd-legacy/stable-diffusion-v1-5"
model_dropdown = ['stabilityai/sdxl-turbo', 'CompVis/stable-diffusion-v1-4', 'sd-legacy/stable-diffusion-v1-5']
model_lora_default = "lora"


def get_lora_sd_pipeline(
    ckpt_dir='./' + model_lora_default,
    base_model_name_or_path=None,
    dtype=torch.float16,
    device=DEVICE,
    adapter_name="default",
    controlnet=None,
    ip_adapter=None
):
    unet_sub_dir = os.path.join(ckpt_dir, "unet")
    text_encoder_sub_dir = os.path.join(ckpt_dir, "text_encoder")
    if os.path.exists(text_encoder_sub_dir) and base_model_name_or_path is None:
        config = LoraConfig.from_pretrained(text_encoder_sub_dir)
        base_model_name_or_path = config.base_model_name_or_path

    if base_model_name_or_path is None:
        raise ValueError("Please specify the base model name or path")


    if controlnet and ip_adapter:
      print('Pipe with ControlNet and IpAdapter')

      controlnet = ControlNetModel.from_pretrained(
          "lllyasviel/sd-controlnet-canny",
          cache_dir="./models_cache",
          torch_dtype=torch.float16
      )
      pipe = StableDiffusionControlNetPipeline.from_pretrained(
         base_model_name_or_path, 
         torch_dtype=dtype, 
         controlnet=controlnet).to(device)
      
      pipe.load_ip_adapter(
          "h94/IP-Adapter",
          subfolder="models",
          weight_name="ip-adapter-plus_sd15.bin",
      )


    elif controlnet:
      print('Pipe with ControlNet')
      controlnet = ControlNetModel.from_pretrained(
          "lllyasviel/sd-controlnet-canny",
          cache_dir="./models_cache",
          torch_dtype=torch.float16)
      pipe = StableDiffusionControlNetPipeline.from_pretrained(base_model_name_or_path, torch_dtype=dtype, controlnet=controlnet)


    elif ip_adapter:
      print('Pipe with IpAdapter')
      pipe = StableDiffusionPipeline.from_pretrained(base_model_name_or_path, torch_dtype=dtype)
      pipe.load_ip_adapter(
          "h94/IP-Adapter",
          subfolder="models",
          weight_name="ip-adapter-plus_sd15.bin")


    else:
      print('Pipe with only SD')
      pipe = StableDiffusionPipeline.from_pretrained(base_model_name_or_path, torch_dtype=dtype)


    pipe.unet = PeftModel.from_pretrained(pipe.unet, unet_sub_dir, adapter_name=adapter_name)
    if os.path.exists(text_encoder_sub_dir):
        pipe.text_encoder = PeftModel.from_pretrained(
            pipe.text_encoder, text_encoder_sub_dir, adapter_name=adapter_name
        )

    if dtype in (torch.float16, torch.bfloat16):
        pipe.unet.half()
        pipe.text_encoder.half()

    pipe.safety_checker = None
    pipe.to(device)
    return pipe


@spaces.GPU
def infer(
    prompt,
    negative_prompt,
    randomize_seed,
    width=512,
    height=512,
    model_repo_id=model_id_default, # в get_lora_sd_pipeline - base_model_name_or_path
    seed=22,
    guidance_scale=7,
    num_inference_steps=50,

    use_advanced_controlnet=False,
    control_strength=None,
    image_upload_cn=None,

    use_advanced_ip=False,
    ip_adapter_scale=None,
    image_upload_ip=None,
    
    model_lora_id=model_lora_default,
    progress=gr.Progress(track_tqdm=True),
    dtype=torch.float16,
    device=DEVICE,
    ):
        
    if randomize_seed:
        seed = random.randint(0, MAX_SEED)
    generator = torch.Generator().manual_seed(seed)


    print(use_advanced_controlnet, use_advanced_ip) 


    if use_advanced_controlnet == False and use_advanced_ip == False:
        print("1. SD 1.5 + Lora")
        pipe = get_lora_sd_pipeline(base_model_name_or_path=model_repo_id, 
                                    dtype=dtype).to(device)                   
        
        image = pipe(prompt,
                num_inference_steps=num_inference_steps,
                guidance_scale=guidance_scale,
                negative_prompt=negative_prompt,
                width=width,
                heigth=height,
                generator=generator).images[0]

    elif use_advanced_controlnet != False and use_advanced_ip == False:
        print("SD 1.5 + Lora + Controlnet")

        edges = cv.Canny(image_upload_cn, 80, 160)
        edges = np.repeat(edges[:, :, None], 3, axis=2)
        edges = Image.fromarray(edges)

        pipe = get_lora_sd_pipeline(base_model_name_or_path=model_repo_id,
                                    controlnet=True,
                                    dtype=dtype).to(device)
        
        image = pipe(prompt,
                edges,
                num_inference_steps = num_inference_steps,
                controlnet_conditioning_scale=control_strength,
                negative_prompt=negative_prompt,
                generator=generator).images[0]
 

    elif use_advanced_ip != False and use_advanced_controlnet == False:
        print("SD 1.5 + Lora + IpAdapter")
        pipe = get_lora_sd_pipeline(base_model_name_or_path=model_repo_id,
                                    ip_adapter=True,
                                    dtype=dtype).to(device)
        pipe.set_ip_adapter_scale(ip_adapter_scale)

        image = pipe(
            prompt,
            ip_adapter_image=image_upload_ip,
            num_inference_steps=num_inference_steps,
            guidance_scale=guidance_scale,
            generator=generator).images[0]

    elif use_advanced_ip != False and use_advanced_controlnet != False:
        print("SD 1.5 + Lora + IpAdapter + ControlNet")

        edges = cv.Canny(image_upload_cn, 80, 160)
        edges = np.repeat(edges[:, :, None], 3, axis=2)
        edges = Image.fromarray(edges)

        pipe = get_lora_sd_pipeline(base_model_name_or_path=model_repo_id,
                                ip_adapter=True,
                                controlnet=True,
                                dtype=dtype).to(device)
        
        
        pipe.set_ip_adapter_scale(ip_adapter_scale)
        image = pipe(prompt,
            edges,
            ip_adapter_image=image_upload_ip,
            num_inference_steps=num_inference_steps,
            guidance_scale=guidance_scale,
            controlnet_conditioning_scale=control_strength,
            height=height,
            width=width,
            generator=generator,
        ).images[0]

    return image, seed