Stable Diffusion (Diffusers) / Google Colab の環境でControlNet 1.1 を使ってバッチ処理で画像を作成する#2 txt2img

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
#
from diffusers import UniPCMultistepScheduler
from diffusers.models import AutoencoderKL
from diffusers.utils import load_image
#import torch.utils
#from controlnet_aux import PidiNetDetector,HEDdetector, MidasDetector, MLSDdetector, OpenposeDetector, PidiNetDetector, NormalBaeDetector, LineartDetector, LineartAnimeDetector, CannyDetector, ContentShuffleDetector, ZoeDetector, MediapipeFaceDetector, SamDetector, LeresDetector
from controlnet_aux.processor import Processor

from PIL import Image

#import cv2
#import numpy as np
from natsort import natsorted
from huggingface_hub import HfApi
from pathlib import Path

import torch
import datetime
import os
import random
import glob

#device = "cuda"
#device = "cpu"
if device == "cpu":
  torch_dtype=torch.float32
else:
  torch_dtype=torch.float16
  device = "cuda"
#
#init_img = load_image(    "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png")
file_format = "%Y%m%d_%H%M%S"
file_list = glob.glob(os.path.join(load_path, "*.png"))
file_list.extend(glob.glob(os.path.join(load_path, "*.jpg")))

#vae = AutoencoderKL.from_pretrained(vae)

#画像生成に使うスケジューラー
#scheduler = EulerDiscreteScheduler.from_pretrained(model_id, subfolder="scheduler")
#scheduler = DPMSolverMultistepScheduler.from_pretrained(model_id, subfolder="scheduler")

#canny = CannyDetector()
#openpose = OpenposeDetector.from_pretrained('lllyasviel/ControlNet')
#hed = HEDdetector.from_pretrained('lllyasviel/Annotators')
#hed = HEDdetector.from_pretrained('lllyasviel/ControlNet')
processor = Processor(controlnet_preprocessor_id)


#controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-openpose", torch_dtype=torch.float16)
#controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-scribble", torch_dtype=torch.float16)
#controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16)
#
controlnet = ControlNetModel.from_pretrained(controlnet_processor_id, torch_dtype=torch_dtype)

#パイプラインの作成
pipe = StableDiffusionControlNetPipeline.from_pretrained(model_id,                                                      #scheduler=scheduler,
                                                      controlnet=controlnet,
                                                      #vae=vae,
                                                      #custom_pipeline="lpw_stable_diffusion",
                                                      safety_checker=None,
                                                      torch_dtype=torch_dtype)
#
pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config)
pipe.enable_model_cpu_offload()
pipe.to(device)

#NSFW規制を無効化する
if pipe.safety_checker is not None:
  pipe.safety_checker = lambda images, **kwargs: (images, False)


if seed is None or seed == -1:
  init_Seed = random.randint(0, 2147483647)
else:
  init_Seed = seed

for img_path in natsorted(file_list):
  infile_name = os.path.basename(img_path)
  infile_name_no_extension = os.path.splitext(infile_name)[0]
  open_img = Image.open(img_path)
  #  open_img = Image.open(img_path).convert("RGB")

  #  canny_image = canny(open_img)     
  #  openpose_image = openpose(open_img)
  #  scribble_image =  hed(open_img, scribble=True)
  init_img =  processor(open_img)
  controlnet_save_path = f"/content/output/controlnet"

  controlnet_image_name = infile_name_no_extension + f".png"
  controlnet_save_pathname = os.path.join(controlnet_save_path, controlnet_image_name)
  #
  init_img.save(controlnet_save_pathname)
  idx = 0

  # 現在の日本時間を取得
  jst_dattetime = datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=9)))

  while idx  < int(batch_count):
   #generator
   mSeed = init_Seed + idx
   generator = torch.Generator(device=device).manual_seed(mSeed)
  #images = []

   #画像を生成
   image = pipe(prompt,
                init_img,
                negative_prompt=negative_prompt,
                width=width, height=height, generator=generator,
                controlnet_conditioning_scale=controlnet_conditioning_scale,
                guidance_scale=CFG_scale, num_inference_steps=Steps
                ).images[0]   
   #outfile_name = (jst_dattetime.strftime(file_format)+ "_" + str(mSeed)+ "-" + str(idx))
   outfile_name = (infile_name_no_extension+ "_" + str(mSeed)+ "-" + str(idx))
   image_name = outfile_name + f".png"

   #画像を保存する
   save_pathname = os.path.join(save_path, image_name)
   image.save(save_pathname)
   idx += 1