import os import json import torch import gc import numpy as np import gradio as gr from PIL import Image from diffusers import StableDiffusionXLPipeline import open_clip from huggingface_hub import hf_hub_download from IP_Composer.IP_Adapter.ip_adapter import IPAdapterXL from IP_Composer.perform_swap import compute_dataset_embeds_svd, get_modified_images_embeds_composition from IP_Composer.generate_text_embeddings import load_descriptions, generate_embeddings import spaces import random device = "cuda" if torch.cuda.is_available() else "cpu" # Initialize SDXL pipeline base_model_path = "stabilityai/stable-diffusion-xl-base-1.0" pipe = StableDiffusionXLPipeline.from_pretrained( base_model_path, torch_dtype=torch.float16, add_watermarker=False, ) # Initialize IP-Adapter image_encoder_repo = 'h94/IP-Adapter' image_encoder_subfolder = 'models/image_encoder' ip_ckpt = hf_hub_download('h94/IP-Adapter', subfolder="sdxl_models", filename='ip-adapter_sdxl_vit-h.bin') ip_model = IPAdapterXL(pipe, image_encoder_repo, image_encoder_subfolder, ip_ckpt, device) # Initialize CLIP model clip_model, _, preprocess = open_clip.create_model_and_transforms('hf-hub:laion/CLIP-ViT-H-14-laion2B-s32B-b79K') clip_model.to(device) tokenizer = open_clip.get_tokenizer('hf-hub:laion/CLIP-ViT-H-14-laion2B-s32B-b79K') CONCEPTS_MAP={ "age": "age_descriptions.npy", "animal fur": "fur_descriptions.npy", "dogs": "dog_descriptions.npy", "emotions": "emotion_descriptions.npy", "flowers": "flower_descriptions.npy", "fruit/vegtable": "fruit_vegetable_descriptions.npy", "outfit type": "outfit_descriptions.npy", "outfit pattern (including color)": "outfit_pattern_descriptions.npy", "patterns": "pattern_descriptions.npy", "patterns (including color)": "pattern_descriptions_with_colors.npy", "vehicle": "vehicle_descriptions.npy", "daytime": "times_of_day_descriptions.npy", "pose": "person_poses_descriptions.npy", "season": "season_descriptions.npy", "material": "material_descriptions_with_gems.npy" } RANKS_MAP={ "age": 30, "animal fur": 80, "dogs": 30, "emotions": 30, "flowers": 30, "fruit/vegtable": 30, "outfit type": 30, "outfit pattern (including color)": 80, "patterns": 80, "patterns (including color)": 80, "vehicle": 30, "daytime": 30, "pose": 30, "season": 30, "material": 80, } concept_options = list(CONCEPTS_MAP.keys()) examples = [ ['./IP_Composer/assets/patterns/base.jpg', './IP_Composer/assets/patterns/pattern.png', 'patterns (including color)', None, None, None, None, 80, 30, 30, None,1.0,0, 30], ['./IP_Composer/assets/flowers/base.png', './IP_Composer/assets/flowers/concept.png', 'flowers', None, None, None, None, 30, 30, 30, None,1.0,0, 30], ['./IP_Composer/assets/materials/base.png', './IP_Composer/assets/materials/concept.jpg', 'material', None, None, None, None, 80, 30, 30, None,1.0,0, 30], ['./IP_Composer/assets/vehicle/base.png', './IP_Composer/assets/vehicle/concept.png', 'vehicle', None, None, None, None, 30, 30, 30, None,1.0,0, 30], ['./IP_Composer/assets/dog_daytime/base.png', './IP_Composer/assets/dog_daytime/daytime.png', 'daytime', './IP_Composer/assets/dog_daytime/dog.png', 'dogs', None, None, 30, 140, 30, None,1.0,0, 30], ['./IP_Composer/assets/pose_material/base.png', './IP_Composer/assets/pose_material/material.jpg', 'material', './IP_Composer/assets/pose_material/pose.png', 'pose', None, None, 30, 80, 30, None,1.0,0, 30], ['./IP_Composer/assets/objects/mug.png', './IP_Composer/assets/patterns/splash.png', 'patterns (including color)', None, None, None, None, 80, 30, 30, None,1.0,0, 30], ['./IP_Composer/assets/objects/mug.png', './IP_Composer/assets/patterns/red_pattern.png', 'patterns (including color)', None, None, None, None, 100, 30, 30, None,1.0,0, 30], ['./IP_Composer/assets/emotions/joyful.png', './IP_Composer/assets/emotions/sad.png', 'emotions', './IP_Composer/assets/age/kid.png', 'age', None, None, 30, 30, 30, None,1.0,0, 30], ['./IP_Composer/assets/flowers/rose_1.jpg', './IP_Composer/assets/flowers/flowers_3.jpg', 'flowers', None, None, None, None, 30, 30, 30, None,1.0,0, 30], ] def generate_examples(base_image, concept_image1, concept_name1, concept_image2, concept_name2, concept_image3, concept_name3, rank1, rank2, rank3, prompt, scale, seed, num_inference_steps): return process_and_display(base_image, concept_image1, concept_name1, concept_image2, concept_name2, concept_image3, concept_name3, rank1, rank2, rank3, prompt, scale, seed, num_inference_steps) MAX_SEED = np.iinfo(np.int32).max def randomize_seed_fn(seed: int, randomize_seed: bool) -> int: if randomize_seed: seed = random.randint(0, MAX_SEED) return seed def change_rank_default(concept_name): return RANKS_MAP.get(concept_name, 30) @spaces.GPU def match_image_to_concept(image): """ Match an uploaded image to the closest concept type using CLIP embeddings """ if image is None: return None # Get image embeddings img_pil = Image.fromarray(image).convert("RGB") img_embed = get_image_embeds(img_pil, clip_model, preprocess, device) # Calculate similarity to each concept similarities = {} for concept_name, concept_file in CONCEPTS_MAP.items(): try: # Load concept embeddings embeds_path = f"./IP_Composer/text_embeddings/{concept_file}" with open(embeds_path, "rb") as f: concept_embeds = np.load(f) # Calculate similarity to each text embedding sim_scores = [] for embed in concept_embeds: # Normalize both embeddings img_embed_norm = img_embed / np.linalg.norm(img_embed) text_embed_norm = embed / np.linalg.norm(embed) # Calculate cosine similarity similarity = np.dot(img_embed_norm.flatten(), text_embed_norm.flatten()) sim_scores.append(similarity) # Use the average of top 5 similarities for better matching sim_scores.sort(reverse=True) top_similarities = sim_scores[:min(5, len(sim_scores))] avg_similarity = sum(top_similarities) / len(top_similarities) similarities[concept_name] = avg_similarity except Exception as e: print(f"Error processing concept {concept_name}: {e}") # Return the concept with highest similarity if similarities: matched_concept = max(similarities.items(), key=lambda x: x[1])[0] # Display a notification to the user gr.Info(f"Image automatically matched to concept: {matched_concept}") return matched_concept return None @spaces.GPU def get_image_embeds(pil_image, model=clip_model, preproc=preprocess, dev=device): """Get CLIP image embeddings for a given PIL image""" image = preproc(pil_image)[np.newaxis, :, :, :] with torch.no_grad(): embeds = model.encode_image(image.to(dev)) return embeds.cpu().detach().numpy() @spaces.GPU def process_images( base_image, concept_image1, concept_name1, concept_image2=None, concept_name2=None, concept_image3=None, concept_name3=None, rank1=10, rank2=10, rank3=10, prompt=None, scale=1.0, seed=420, num_inference_steps=50, concpet_from_file_1 = None, concpet_from_file_2 = None, concpet_from_file_3 = None, use_concpet_from_file_1 = False, use_concpet_from_file_2 = False, use_concpet_from_file_3 = False ): """Process the base image and concept images to generate modified images""" # Process base image base_image_pil = Image.fromarray(base_image).convert("RGB") base_embed = get_image_embeds(base_image_pil, clip_model, preprocess, device) # Process concept images concept_images = [] concept_descriptions = [] skip_load_concept =[False,False, False] # for demo purposes we allow for up to 3 different concepts and corresponding concept images if concept_image1 is not None: concept_images.append(concept_image1) if use_concpet_from_file_1 and concpet_from_file_1 is not None: # if concept is new from user input concept_descriptions.append(concpet_from_file_1) skip_load_concept[0] = True else: concept_descriptions.append(CONCEPTS_MAP[concept_name1]) else: return None, "Please upload at least one concept image" # Add second concept (optional) if concept_image2 is not None: concept_images.append(concept_image2) if use_concpet_from_file_2 and concpet_from_file_2 is not None: # if concept is new from user input concept_descriptions.append(concpet_from_file_2) skip_load_concept[1] = True else: concept_descriptions.append(CONCEPTS_MAP[concept_name2]) # Add third concept (optional) if concept_image3 is not None: concept_images.append(concept_image3) if use_concpet_from_file_3 and concpet_from_file_3 is not None: # if concept is new from user input concept_descriptions.append(concpet_from_file_3) skip_load_concept[2] = True else: concept_descriptions.append(CONCEPTS_MAP[concept_name3]) # Get all ranks ranks = [rank1] if concept_image2 is not None: ranks.append(rank2) if concept_image3 is not None: ranks.append(rank3) concept_embeds = [] projection_matrices = [] # for the demo, we assume 1 concept image per concept # for each concept image, we calculate it's image embeedings and load the concepts textual embeddings to copmpute the projection matrix over it for i, concept in enumerate(concept_descriptions): img_pil = Image.fromarray(concept_images[i]).convert("RGB") concept_embeds.append(get_image_embeds(img_pil, clip_model, preprocess, device)) if skip_load_concept[i]: # if concept is new from user input all_embeds_in = concept else: embeds_path = f"./IP_Composer/text_embeddings/{concept}" with open(embeds_path, "rb") as f: all_embeds_in = np.load(f) projection_matrix = compute_dataset_embeds_svd(all_embeds_in, ranks[i]) projection_matrices.append(projection_matrix) # Create projection data structure for the composition projections_data = [ { "embed": embed, "projection_matrix": proj_matrix } for embed, proj_matrix in zip(concept_embeds, projection_matrices) ] # Generate modified images - modified_images = get_modified_images_embeds_composition( base_embed, projections_data, ip_model, prompt=prompt, scale=scale, num_samples=1, seed=seed, num_inference_steps=num_inference_steps ) return modified_images[0] @spaces.GPU def get_text_embeddings(concept_file): print("generating text embeddings") descriptions = load_descriptions(concept_file) embeddings = generate_embeddings(descriptions, clip_model, tokenizer, device, batch_size=100) print("text embeddings shape",embeddings.shape) return embeddings, True def process_and_display( base_image, concept_image1, concept_name1="age", concept_image2=None, concept_name2=None, concept_image3=None, concept_name3=None, rank1=30, rank2=30, rank3=30, prompt=None, scale=1.0, seed=0, num_inference_steps=50, concpet_from_file_1 = None, concpet_from_file_2 = None, concpet_from_file_3 = None, use_concpet_from_file_1 = False, use_concpet_from_file_2 = False, use_concpet_from_file_3 = False ): if base_image is None: raise gr.Error("Please upload a base image") if concept_image1 is None: raise gr.Error("Choose at least one concept image") if concept_image1 is None: raise gr.Error("Choose at least one concept type") modified_images = process_images( base_image, concept_image1, concept_name1, concept_image2, concept_name2, concept_image3, concept_name3, rank1, rank2, rank3, prompt, scale, seed, num_inference_steps, concpet_from_file_1, concpet_from_file_2, concpet_from_file_3, use_concpet_from_file_1, use_concpet_from_file_2, use_concpet_from_file_3 ) return modified_images # UI CSS css = """ #col-container { margin: 0 auto; max-width: 800px; } .gradio-container{ max-width: 1024px !important; margin: 0 auto } """ example = """ Emotion Description a photo of a person feeling joyful a photo of a person feeling sorrowful a photo of a person feeling enraged a photo of a person feeling astonished a photo of a person feeling disgusted a photo of a person feeling terrified ... """ with gr.Blocks(css=css) as demo: gr.Markdown(f"""# IP Composer 🌅✚🖌️ ### compose new images with visual concepts extracted from refrence images using CLIP & IP Adapter #### 🛠️ How to Use: 1. Upload a base image 2. Upload 1–3 concept images 3. Select a **concept type** to extract from each concept image: - Choose a **predefined concept type** from the dropdown (e.g. pattern, emotion, pose), **or** - Upload a **file with text variations of your concept** (e.g. prompts from an LLM). - 👉 If you're uploading a **new concept**, don't forget to **adjust the "rank" value** under **Advanced Options** for better results. Following the algorithm proposed in IP-Composer: Semantic Composition of Visual Concepts by Dorfman et al. [[Project page](https://ip-composer.github.io/IP-Composer/)] [[arxiv](https://arxiv.org/pdf/2502.13951)] """) concpet_from_file_1 = gr.State() concpet_from_file_2 = gr.State() concpet_from_file_3 = gr.State() use_concpet_from_file_1 = gr.State() use_concpet_from_file_2 = gr.State() use_concpet_from_file_3 = gr.State() with gr.Row(): with gr.Column(): base_image = gr.Image(label="Base Image (Required)", type="numpy", height=400, width=400) with gr.Tab("Concept 1"): with gr.Group(): concept_image1 = gr.Image(label="Concept Image 1", type="numpy", height=400, width=400) with gr.Column(): concept_name1 = gr.Dropdown(concept_options, label="Concept 1", value=None, info="Pick concept type") with gr.Accordion("💡 Or use a new concept 👇", open=False): gr.Markdown("1. Upload a file with text variations of your concept (e.g. ask an LLM)") gr.Markdown("2. Prefereably with > 100 variations.") with gr.Accordion("File example for the concept 'emotions'", open=False): gr.Markdown(example) concept_file_1 = gr.File(label="Concept variations", file_types=["text"]) with gr.Tab("Concept 2 (Optional)"): with gr.Group(): concept_image2 = gr.Image(label="Concept Image 2", type="numpy", height=400, width=400) with gr.Column(): concept_name2 = gr.Dropdown(concept_options, label="Concept 2", value=None, info="Pick concept type") with gr.Accordion("💡 Or use a new concept 👇", open=False): gr.Markdown("1. Upload a file with text variations of your concept (e.g. ask an LLM)") gr.Markdown("2. Prefereably with > 100 variations.") with gr.Accordion("File example for the concept 'emotions'", open=False): gr.Markdown(example) concept_file_2 = gr.File(label="Concept variations", file_types=["text"]) with gr.Tab("Concept 3 (optional)"): with gr.Group(): concept_image3 = gr.Image(label="Concept Image 3", type="numpy", height=400, width=400) with gr.Column(): concept_name3 = gr.Dropdown(concept_options, label="Concept 3", value= None, info="Pick concept type") with gr.Accordion("💡 Or use a new concept 👇", open=False): gr.Markdown("1. Upload a file with text variations of your concept (e.g. ask an LLM)") gr.Markdown("2. Prefereably with > 100 variations.") with gr.Accordion("File example for the concept 'emotions'", open=False): gr.Markdown(example) concept_file_3 = gr.File(label="Concept variations", file_types=["text"]) with gr.Accordion("Advanced options", open=False): prompt = gr.Textbox(label="Guidance Prompt (Optional)", placeholder="Optional text prompt to guide generation") num_inference_steps = gr.Slider(minimum=1, maximum=50, value=30, step=1, label="Num steps") with gr.Row(): scale = gr.Slider(minimum=0.1, maximum=2.0, value=1.0, step=0.1, label="Scale") randomize_seed = gr.Checkbox(value=True, label="Randomize seed") seed = gr.Number(value=0, label="Seed", precision=0) with gr.Column(): gr.Markdown("If a concept is not showing enough, try to increase the rank") with gr.Row(): rank1 = gr.Slider(minimum=1, maximum=150, value=30, step=1, label="Rank concept 1") rank2 = gr.Slider(minimum=1, maximum=150, value=30, step=1, label="Rank concept 2") rank3 = gr.Slider(minimum=1, maximum=150, value=30, step=1, label="Rank concept 3") with gr.Column(): output_image = gr.Image(label="Composed output", show_label=True,height=400, width=400 ) submit_btn = gr.Button("Generate") gr.Examples( examples, inputs=[base_image, concept_image1, concept_name1, concept_image2, concept_name2, concept_image3, concept_name3, rank1, rank2, rank3, prompt, scale, seed, num_inference_steps], outputs=[output_image], fn=generate_examples, cache_examples=False ) concept_file_1.upload( fn=get_text_embeddings, inputs=[concept_file_1], outputs=[concpet_from_file_1, use_concpet_from_file_1] ) concept_file_2.upload( fn=get_text_embeddings, inputs=[concept_file_2], outputs=[concpet_from_file_2, use_concpet_from_file_2] ) concept_file_3.upload( fn=get_text_embeddings, inputs=[concept_file_3], outputs=[concpet_from_file_3, use_concpet_from_file_3] ) concept_file_1.delete( fn=lambda x: False, inputs=[concept_file_1], outputs=[use_concpet_from_file_1] ) concept_file_2.delete( fn=lambda x: False, inputs=[concept_file_2], outputs=[use_concpet_from_file_2] ) concept_file_3.delete( fn=lambda x: False, inputs=[concept_file_3], outputs=[use_concpet_from_file_3] ) submit_btn.click( fn=randomize_seed_fn, inputs=[seed, randomize_seed], outputs=seed, ).then(fn=process_and_display, inputs=[ base_image, concept_image1, concept_name1, concept_image2, concept_name2, concept_image3, concept_name3, rank1, rank2, rank3, prompt, scale, seed, num_inference_steps, concpet_from_file_1, concpet_from_file_2, concpet_from_file_3, use_concpet_from_file_1, use_concpet_from_file_2, use_concpet_from_file_3 ], outputs=[output_image] ) concept_name1.select( fn= change_rank_default, inputs=[concept_name1], outputs=[rank1] ) concept_name2.select( fn= change_rank_default, inputs=[concept_name2], outputs=[rank2] ) concept_name3.select( fn= change_rank_default, inputs=[concept_name3], outputs=[rank3] ) concept_image1.upload( fn=match_image_to_concept, inputs=[concept_image1], outputs=[concept_name1] ) concept_image2.upload( fn=match_image_to_concept, inputs=[concept_image2], outputs=[concept_name2] ) concept_image3.upload( fn=match_image_to_concept, inputs=[concept_image3], outputs=[concept_name3] ) if __name__ == "__main__": demo.launch()