wuutiing2/flux_controlnet_training_example

this is a debug trainning result for diffusers training script

related issue https://github.com/huggingface/diffusers/issues/11181

related wandb log https://wandb.ai/wuuutiiing/flux_train_controlnet/overview

related script https://github.com/huggingface/diffusers/blob/main/examples/controlnet/train_controlnet_flux.py

test script

from diffusers import FluxControlNetModel, FluxControlNetPipeline
import torch
from PIL import Image

validation_image = Image.open("a_control_image_like_fill50k.png").convert("RGB")

control_path = "wuutiing2/flux_controlnet_training_example"
flux_path = "black-forest-labs/FLUX.1-dev"

flux_controlnet = FluxControlNetModel.from_pretrained(
    control_path, torch_dtype=torch.bfloat16
)
pipeline = FluxControlNetPipeline.from_pretrained(
    flux_path,
    controlnet=flux_controlnet,
    torch_dtype=torch.bfloat16,
        )
pipeline.to("cuda")

image = pipeline(
    prompt="aqua circle with light pink background",
    control_image=validation_image,
    num_inference_steps=28,
    controlnet_conditioning_scale=1,
    guidance_scale=3.5,
    generator=torch.manual_seed(19),
).images[0]

image.save("output.png")