import json import os import types from urllib.parse import urlparse import cv2 import diffusers import gradio as gr import numpy as np import spaces import torch from einops import rearrange from huggingface_hub import hf_hub_download from omegaconf import OmegaConf from PIL import Image, ImageOps from safetensors.torch import load_file from torch.nn import functional as F from torchdiffeq import odeint_adjoint as odeint from echoflow.common import instantiate_class_from_config, unscale_latents from echoflow.common.models import ( ContrastiveModel, DiffuserSTDiT, ResNet18, SegDiTTransformer2DModel, ) torch.set_grad_enabled(False) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") dtype = torch.float32 print(f"Using device: {device}") # 4f4 latent space B, T, C, H, W = 1, 64, 4, 28, 28 VIEWS = ["A4C", "PSAX", "PLAX"] def load_model(path): if path.startswith("http"): parsed_url = urlparse(path) if "huggingface.co" in parsed_url.netloc: parts = parsed_url.path.strip("/").split("/") repo_id = "/".join(parts[:2]) subfolder = None if len(parts) > 3: subfolder = "/".join(parts[4:]) local_root = "./tmp" local_dir = os.path.join(local_root, repo_id.replace("/", "_")) if subfolder: local_dir = os.path.join(local_root, subfolder) os.makedirs(local_root, exist_ok=True) config_file = hf_hub_download( repo_id=repo_id, subfolder=subfolder, filename="config.json", local_dir=local_root, repo_type="model", token=os.getenv("READ_HF_TOKEN"), local_dir_use_symlinks=False, ) assert os.path.exists(config_file) hf_hub_download( repo_id=repo_id, filename="diffusion_pytorch_model.safetensors", subfolder=subfolder, local_dir=local_root, local_dir_use_symlinks=False, token=os.getenv("READ_HF_TOKEN"), ) path = local_dir model_root = os.path.join(config_file.split("config.json")[0]) json_path = os.path.join(model_root, "config.json") assert os.path.exists(json_path) with open(json_path, "r") as f: config = json.load(f) klass_name = config["_class_name"] klass = getattr(diffusers, klass_name, None) or globals().get(klass_name, None) assert ( klass is not None ), f"Could not find class {klass_name} in diffusers or global scope." assert hasattr( klass, "from_pretrained" ), f"Class {klass_name} does not support 'from_pretrained'." return klass.from_pretrained(path) def load_reid(path): parsed_url = urlparse(path) parts = parsed_url.path.strip("/").split("/") repo_id = "/".join(parts[:2]) subfolder = "/".join(parts[4:]) local_root = "./tmp" config_file = hf_hub_download( repo_id=repo_id, subfolder=subfolder, filename="config.yaml", local_dir=local_root, repo_type="model", token=os.getenv("READ_HF_TOKEN"), local_dir_use_symlinks=False, ) weights_file = hf_hub_download( repo_id=repo_id, subfolder=subfolder, filename="backbone.safetensors", local_dir=local_root, repo_type="model", token=os.getenv("READ_HF_TOKEN"), local_dir_use_symlinks=False, ) config = OmegaConf.load(config_file) backbone = instantiate_class_from_config(config.backbone) backbone = ContrastiveModel.patch_backbone( backbone, config.model.args.in_channels, config.model.args.out_channels ) state_dict = load_file(weights_file) backbone.load_state_dict(state_dict) backbone = backbone.to(device, dtype=dtype) backbone.eval() return backbone def get_vae_scaler(path): scaler = torch.load(path) scaler = {k: v.to(device) for k, v in scaler.items()} return scaler # generator = torch.Generator(device=device).manual_seed(0) lifm = load_model("https://huggingface.co./HReynaud/EchoFlow/tree/main/lifm/FMiT-S2-4f4") lifm = lifm.to(device, dtype=dtype) lifm.eval() vae = load_model("https://huggingface.co./HReynaud/EchoFlow/tree/main/vae/avae-4f4") vae = vae.to(device, dtype=dtype) vae.eval() vae_scaler = get_vae_scaler("assets/scaling.pt") reid = { "anatomies": { "A4C": torch.cat( [ torch.load("assets/anatomies_dynamic.pt"), torch.load("assets/anatomies_ped_a4c.pt"), ], dim=0, ), "PSAX": torch.load("assets/anatomies_ped_psax.pt"), "PLAX": torch.load("assets/anatomies_lvh.pt"), }, "models": { "A4C": load_reid( "https://huggingface.co./HReynaud/EchoFlow/tree/main/reid/dynamic-4f4" ), "PSAX": load_reid( "https://huggingface.co./HReynaud/EchoFlow/tree/main/reid/ped_psax-4f4" ), "PLAX": load_reid( "https://huggingface.co./HReynaud/EchoFlow/tree/main/reid/lvh-4f4" ), }, "tau": { "A4C": 0.9997, "PSAX": 0.9997, "PLAX": 0.9997, }, } lvfm = load_model("https://huggingface.co./HReynaud/EchoFlow/tree/main/lvfm/FMvT-S2-4f4") lvfm = lvfm.to(device, dtype=dtype) lvfm.eval() def load_default_mask(): """Load the default mask from disk. If not found, return a blank black mask.""" default_mask_path = os.path.join("assets", "default_mask.png") try: if os.path.exists(default_mask_path): mask = Image.open(default_mask_path).convert("L") # Ensure the mask is square and of proper size mask = mask.resize((400, 400), Image.Resampling.LANCZOS) # Make sure it's binary (0 or 255) mask = ImageOps.autocontrast(mask, cutoff=0) return np.array(mask) except Exception as e: print(f"Error loading default mask: {e}") # Return a blank black mask if no default mask is found return np.zeros((400, 400), dtype=np.uint8) def preprocess_mask(mask): """Ensure mask is properly formatted for the model.""" if mask is None: return np.zeros((112, 112), dtype=np.uint8) # Check if mask is an EditorValue with multiple parts if isinstance(mask, dict) and "composite" in mask: # Use the composite image from the ImageEditor mask = mask["composite"] # If mask is already a numpy array, convert to PIL for processing if isinstance(mask, np.ndarray): mask_pil = Image.fromarray(mask) else: mask_pil = mask # Ensure the mask is in L mode (grayscale) mask_pil = mask_pil.convert("L") # Apply contrast to make it binary (0 or 255) mask_pil = ImageOps.autocontrast(mask_pil, cutoff=0) # Threshold to ensure binary values mask_pil = mask_pil.point(lambda p: 255 if p > 127 else 0) # Print sizes for debugging # print(f"Original mask size: {mask_pil.size}") # Resize to 112x112 for the model mask_pil = mask_pil.resize((112, 112), Image.Resampling.LANCZOS) # Convert back to numpy array return np.array(mask_pil) @spaces.GPU(duration=3) @torch.no_grad() def generate_latent_image(mask, class_selection, sampling_steps=50): """Generate a latent image based on mask, class selection, and sampling steps""" # Mask mask = preprocess_mask(mask) mask = torch.from_numpy(mask).to(device, dtype=dtype) mask = mask.unsqueeze(0).unsqueeze(0) mask = F.interpolate(mask, size=(H, W), mode="bilinear", align_corners=False) mask = 1.0 * (mask > 0) # print(mask.shape, mask.min(), mask.max(), mask.mean(), mask.std()) # Class class_idx = VIEWS.index(class_selection) class_idx = torch.tensor([class_idx], device=device, dtype=torch.long) # Timesteps timesteps = torch.linspace( 1.0, 0.0, steps=sampling_steps + 1, device=device, dtype=dtype ) forward_kwargs = { "class_labels": class_idx, # B x 1 "segmentation": mask, # B x 1 x H x W } z_1 = torch.randn( (B, C, H, W), device=device, dtype=dtype, # generator=generator, ) lifm.forward_original = lifm.forward def new_forward(self, t, y, *args, **kwargs): kwargs = {**kwargs, **forward_kwargs} return self.forward_original(y, t.view(1), *args, **kwargs).sample lifm.forward = types.MethodType(new_forward, lifm) # Use odeint to integrate with torch.autocast("cuda"): latent_image = odeint( lifm, z_1, timesteps, atol=1e-5, rtol=1e-5, adjoint_params=lifm.parameters(), method="euler", )[-1] lifm.forward = lifm.forward_original latent_image = latent_image.detach().cpu().numpy() # callm VAE here return latent_image # B x C x H x W @spaces.GPU(duration=3) @torch.no_grad() def decode_images(latents): """Decode latent representations to pixel space using a VAE. Args: latents: A numpy array of shape [B, C, H, W] for single image or [B, C, T, H, W] for sequences/animations Returns: numpy array of decoded images in [B, H, W, 3] format for single image or [B, C, T, H, W] for sequences """ global vae if latents is None: return None vae = vae.to(device, dtype=dtype) vae.eval() # Convert to torch tensor if needed if not isinstance(latents, torch.Tensor): latents = torch.from_numpy(latents).to(device, dtype=dtype) # Unscale latents latents = unscale_latents(latents, vae_scaler) # Handle both single images and sequences is_sequence = len(latents.shape) == 5 # B C T H W # print("Sequence:", is_sequence) if is_sequence: B, C, T, H, W = latents.shape latents = rearrange(latents[0], "c t h w -> t c h w") else: B, C, H, W = latents.shape # print("Latents:", latents.shape) with torch.no_grad(): # Decode latents to pixel space # decode one by one decoded = [] for i in range(latents.shape[0]): decoded.append(vae.decode(latents[i : i + 1].float()).sample) decoded = torch.cat(decoded, dim=0) decoded = (decoded + 1) * 128 decoded = decoded.clamp(0, 255).to(torch.uint8).cpu() if is_sequence: # Reshape back to [B, C, T, H, W] for sequences decoded = rearrange(decoded, "t c h w -> c t h w").unsqueeze(0) else: decoded = decoded.squeeze() decoded = decoded.permute(1, 2, 0) # print("Decoded:", decoded.shape) return decoded.numpy() def decode_latent_to_pixel(latent_image): """Decode a single latent image to pixel space""" if latent_image is None: return None # Add batch dimension if needed if len(latent_image.shape) == 3: latent_image = latent_image[None, ...] decoded_image = decode_images(latent_image) decoded_image = cv2.resize( decoded_image, (400, 400), interpolation=cv2.INTER_NEAREST ) return decoded_image @spaces.GPU(duration=3) @torch.no_grad() def check_privacy(latent_image_numpy, class_selection): """Check if the latent image is too similar to database images""" latent_image = torch.from_numpy(latent_image_numpy).to(device, dtype=dtype) reid_model = reid["models"][class_selection].to(device, dtype=dtype) real_anatomies = reid["anatomies"][class_selection] # already scaled tau = reid["tau"][class_selection] with torch.no_grad(): features = reid_model(latent_image).sigmoid().cpu() corr = torch.corrcoef(torch.cat([real_anatomies, features], dim=0))[0, 1:] corr = corr.max() if corr > tau: return ( None, f"⚠️ **Warning:** Generated image is too similar to training data. Privacy check failed.", ) else: return ( latent_image_numpy, f"✅ **Success:** Generated image passed privacy check.", ) @spaces.GPU(duration=3) @torch.no_grad() def generate_animation( latent_image, ejection_fraction, sampling_steps=50, cfg_scale=1.0 ): """Generate an animated sequence of latent images based on EF""" # print( # f"Generating animation with EF = {ejection_fraction}, steps = {sampling_steps}, CFG = {cfg_scale}" # ) # print(latent_image.shape, type(latent_image)) print("Generating animation...") if latent_image is None: return None lvefs = torch.tensor([ejection_fraction / 100.0], device=device, dtype=dtype) lvefs = lvefs[:, None, None].to(device, dtype) uncond_lvefs = -1 * torch.ones_like(lvefs) ref_images = torch.from_numpy(latent_image).to(device, dtype) ref_images = ref_images[:, :, None, :, :] # B x C x 1 x H x W ref_images = ref_images.repeat(1, 1, T, 1, 1) # B x C x T x H x W uncond_images = torch.zeros_like(ref_images) timesteps = torch.linspace( 1.0, 0.0, steps=sampling_steps + 1, device=device, dtype=dtype ) forward_kwargs = { "encoder_hidden_states": lvefs, "cond_image": ref_images, } z_1 = torch.randn( (B, C, T, H, W), device=device, dtype=dtype, # generator=generator, ) # print( # z_1.shape, # forward_kwargs["encoder_hidden_states"].shape, # forward_kwargs["cond_image"].shape, # ) lvfm.forward_original = lvfm.forward def new_forward(self, t, y, *args, **kwargs): kwargs = {**kwargs, **forward_kwargs} # y has shape (B, C, T, H, W) pred = self.forward_original(y, t.repeat(y.size(0)), *args, **kwargs).sample if cfg_scale != 1.0: uncond_kwargs = { "encoder_hidden_states": uncond_lvefs, "cond_image": uncond_images, } uncond_pred = self.forward_original( y, t.repeat(y.size(0)), *args, **uncond_kwargs ).sample pred = uncond_pred + cfg_scale * (pred - uncond_pred) return pred lvfm.forward = types.MethodType(new_forward, lvfm) with torch.autocast("cuda"): synthetic_video = odeint( lvfm, z_1, timesteps, atol=1e-5, rtol=1e-5, adjoint_params=lvfm.parameters(), method="euler", )[-1] lvfm.forward = lvfm.forward_original # print("Synthetic video:", synthetic_video.shape) print("Animation generated") return synthetic_video.detach().cpu() # B x C x T x H x W @spaces.GPU(duration=3) @torch.no_grad() def decode_animation(latent_animation): """Decode a latent animation to pixel space""" if latent_animation is None: return None # Convert to torch tensor if needed if not isinstance(latent_animation, torch.Tensor): latent_animation = torch.from_numpy(latent_animation) latent_animation = latent_animation.to(device, dtype=dtype) # Ensure shape is B x C x T x H x W if len(latent_animation.shape) == 4: # [T, C, H, W] latent_animation = latent_animation[None, ...] # Add batch dimension # Decode using VAE decoded = decode_images(latent_animation) # Returns B x C x T x H x W numpy array # Remove batch dimension and transpose to T x H x W x C decoded = np.transpose(decoded[0], (1, 2, 3, 0)) # [T, H, W, C] # Resize frames to 400x400 decoded = np.stack( [ cv2.resize(frame, (400, 400), interpolation=cv2.INTER_NEAREST) for frame in decoded ] ) # Save to temporary file temp_file = "temp_video_2.mp4" fps = 32 fourcc = cv2.VideoWriter_fourcc(*"mp4v") out = cv2.VideoWriter(temp_file, fourcc, fps, (400, 400)) # Write frames for frame in decoded: out.write(frame) out.release() return temp_file def convert_latent_to_display(latent_image): """Convert multi-channel latent image to grayscale for display""" if latent_image is None: return None # Check shape if len(latent_image.shape) == 4: # [B, C, H, W] # Remove batch dimension and average across channels display_image = np.squeeze(latent_image, axis=0) # [C, H, W] display_image = np.mean(display_image, axis=0) # [H, W] elif len(latent_image.shape) == 3: # [C, H, W] # Average across channels display_image = np.mean(latent_image, axis=0) # [H, W] else: display_image = latent_image # Normalize to 0-1 range display_image = (display_image - display_image.min()) / ( display_image.max() - display_image.min() + 1e-8 ) # Convert to grayscale image display_image = (display_image * 255).astype(np.uint8) # Resize to a larger size (e.g., 400x400) using bicubic interpolation display_image = cv2.resize( display_image, (400, 400), interpolation=cv2.INTER_NEAREST ) return display_image @spaces.GPU(duration=3) @torch.no_grad() def latent_animation_to_grayscale(latent_animation): """Convert multi-channel latent animation to grayscale for display""" if latent_animation is None: return None # print("Input shape:", latent_animation.shape) # Convert to numpy if it's a torch tensor if torch.is_tensor(latent_animation): latent_animation = latent_animation.detach().cpu().numpy() # Handle shape B x C x T x H x W -> T x H x W if len(latent_animation.shape) == 5: # [B, C, T, H, W] latent_animation = np.squeeze(latent_animation, axis=0) # [C, T, H, W] latent_animation = np.transpose(latent_animation, (1, 0, 2, 3)) # [T, C, H, W] # print("After transpose:", latent_animation.shape) # Average across channels latent_animation = np.mean(latent_animation, axis=1) # [T, H, W] # print("After channel reduction:", latent_animation.shape) # Normalize each frame independently min_vals = latent_animation.min(axis=(1, 2), keepdims=True) max_vals = latent_animation.max(axis=(1, 2), keepdims=True) latent_animation = (latent_animation - min_vals) / (max_vals - min_vals + 1e-8) # Convert to uint8 latent_animation = (latent_animation * 255).astype(np.uint8) # print("Before resize:", latent_animation.shape) # Resize each frame resized_frames = [] for frame in latent_animation: resized = cv2.resize(frame, (400, 400), interpolation=cv2.INTER_NEAREST) resized_frames.append(resized) # Stack back into video grayscale_video = np.stack(resized_frames) # print("Final shape:", grayscale_video.shape) # Add a dummy channel dimension for grayscale video grayscale_video = grayscale_video[..., None].repeat(3, axis=-1) # Convert to RGB # print("Output shape with channels:", grayscale_video.shape) # Save to temporary file temp_file = "temp_video.mp4" fps = 32 # Create VideoWriter object fourcc = cv2.VideoWriter_fourcc(*"mp4v") out = cv2.VideoWriter(temp_file, fourcc, fps, (400, 400)) # Write frames for frame in grayscale_video: out.write(frame) out.release() return temp_file # Add function to load view-specific mask def load_view_mask(view): mask_path = f"assets/{view.lower()}_seg.png" try: mask_image = Image.open(mask_path).convert("L") mask_image = mask_image.resize((400, 400), Image.Resampling.LANCZOS) # Make it binary (0 or 255) mask_image = ImageOps.autocontrast(mask_image, cutoff=0) mask_array = np.array(mask_image) # Create the editor value structure editor_value = { "background": np.zeros((400, 400), dtype=np.uint8), # Black background "layers": [mask_array], # The mask as an editable layer "composite": mask_array, # The composite image } return editor_value except Exception as e: print(f"Error loading mask for view {view}: {e}") return None custom_js = """ """ def create_demo(): black_background = np.zeros((400, 400), dtype=np.uint8) # Load the default mask image if it exists try: mask_image = Image.open("assets/a4c_seg.png").convert("L") mask_image = mask_image.resize((400, 400), Image.Resampling.LANCZOS) # Make it binary (0 or 255) mask_image = ImageOps.autocontrast(mask_image, cutoff=0) mask_image = mask_image.point(lambda p: 255 if p > 127 else 0) mask_array = np.array(mask_image) # Create the editor value structure editor_value = { "background": black_background, # Black background "layers": [mask_array], # The mask as an editable layer "composite": mask_array, # The composite image (what's displayed) } except Exception as e: print(f"Error loading mask image: {e}") # Fall back to empty canvas editor_value = black_background # Define all components first mask_input = gr.ImageEditor( label="Binary Mask", height=400, width=400, image_mode="L", value=editor_value, type="numpy", brush=gr.Brush( colors=["#ffffff"], color_mode="fixed", default_size=20, default_color="#ffffff", ), eraser=gr.Eraser(default_size=20), show_download_button=True, sources=[], canvas_size=(400, 400), fixed_canvas=True, layers=False, render=False, ) class_selection = gr.Radio( choices=["A4C", "PSAX", "PLAX"], label="View Class", value="A4C", render=False, ) sampling_steps = gr.Slider( minimum=1, maximum=200, value=100, step=1, label="Number of Sampling Steps", render=False, ) ef_slider = gr.Slider( minimum=0, maximum=100, value=65, label="Ejection Fraction (%)", render=False, ) animation_steps = gr.Slider( minimum=1, maximum=200, value=100, step=1, label="Number of Sampling Steps.", render=False, ) cfg_slider = gr.Slider( minimum=0, maximum=10, value=1, step=1, label="Classifier-Free Guidance Scale", render=False, ) latent_image_display = gr.Image( label="Latent Image", type="numpy", height=400, width=400, render=False, ) decoded_image_display = gr.Image( label="Decoded Image", type="numpy", height=400, width=400, render=False, ) privacy_status = gr.Markdown(render=False) filtered_latent_display = gr.Image( label="Filtered Latent Image", type="numpy", height=400, width=400, render=False, ) latent_animation_display = gr.Video( label="Latent Video", format="mp4", render=False, autoplay=True, loop=True, ) decoded_animation_display = gr.Video( label="Decoded Video", format="mp4", render=False, autoplay=True, loop=True, ) # Define the theme and layout with gr.Blocks(theme=gr.themes.Soft(), head=custom_js) as demo: gr.Markdown( "# EchoFlow: A Foundation Model for Cardiac Ultrasound Image and Video Generation" ) gr.Markdown("## Preprint: https://arxiv.org/abs/2503.22357") gr.Markdown("## Dataset Generation Pipeline") gr.Markdown( """ This demo showcases EchoFlow's ability to generate synthetic echocardiogram images and videos while preserving patient privacy. The pipeline consists of four main steps: 1. **Latent Image Generation**: Draw a mask to indicate the region where the Left Ventricle should appear. Select the desired cardiac view, and click "Generate Latent Image". This outputs a latent image, which can be decoded into a pixel space image by clicking "Decode to Pixel Space". 2. **Privacy Filter**: When clicking "Run Privacy Check", the generated image will be checked against a database of all training anatomies to ensure it is sufficiently different from real patient data. 3. **Latent Video Generation**: If the privacy check passes, the latent image can be animated into a video with the desired Ejection Fraction. 4. **Video Decoding**: The video can be decoded back to pixel space by clicking "Decode Video". ### ⚙️ Parameters - **Sampling Steps**: Higher values produce better quality but take longer - **Ejection Fraction**: Controls the strength of heart contraction in the animation - **CFG Scale**: Controls how closely the animation follows the specified conditions """ ) def load_example( mask, view, steps, ef, anim_steps, cfg, latent, decoded, status, filtered, latent_vid, decoded_vid, ): # This function will be called when an example is clicked # It returns all values in order they should be loaded into components return [ mask, view, steps, ef, anim_steps, cfg, latent, decoded, status, filtered, latent_vid, decoded_vid, ] # Add examples using the components examples = gr.Examples( examples=[ # Example 1: A4C view [ # Inputs { "background": np.zeros((400, 400), dtype=np.uint8), "layers": [ np.array( Image.open("assets/a4c_seg.png") .convert("L") .resize((400, 400)) ) ], "composite": np.array( Image.open("assets/a4c_seg.png") .convert("L") .resize((400, 400)) ), }, "A4C", # view 100, # sampling steps 65, # EF slider 100, # animation steps 1.0, # cfg scale # Pre-computed outputs Image.open("assets/examples/a4c_latent.png"), # latent image Image.open("assets/examples/a4c_decoded.png"), # decoded image "✅ **Success:** Generated image passed privacy check.", # privacy status Image.open("assets/examples/a4c_filtered.png"), # filtered latent "assets/examples/a4c_latent.mp4", # latent animation "assets/examples/a4c_decoded.mp4", # decoded animation ], # Example 2: PSAX view [ # Inputs { "background": np.zeros((400, 400), dtype=np.uint8), "layers": [ np.array( Image.open("assets/psax_seg.png") .convert("L") .resize((400, 400)) ) ], "composite": np.array( Image.open("assets/psax_seg.png") .convert("L") .resize((400, 400)) ), }, "PSAX", # view 100, # sampling steps 65, # EF slider 100, # animation steps 1.0, # cfg scale # Pre-computed outputs Image.open("assets/examples/psax_latent.png"), # latent image Image.open("assets/examples/psax_decoded.png"), # decoded image "✅ **Success:** Generated image passed privacy check.", # privacy status Image.open("assets/examples/psax_filtered.png"), # filtered latent "assets/examples/psax_latent.mp4", # latent animation "assets/examples/psax_decoded.mp4", # decoded animation ], # Example 3: PLAX view [ # Inputs { "background": np.zeros((400, 400), dtype=np.uint8), "layers": [ np.array( Image.open("assets/plax_seg.png") .convert("L") .resize((400, 400)) ) ], "composite": np.array( Image.open("assets/plax_seg.png") .convert("L") .resize((400, 400)) ), }, "PLAX", # view 100, # sampling steps 65, # EF slider 100, # animation steps 1.0, # cfg scale # Pre-computed outputs Image.open("assets/examples/plax_latent.png"), # latent image Image.open("assets/examples/plax_decoded.png"), # decoded image "✅ **Success:** Generated image passed privacy check.", # privacy status Image.open("assets/examples/plax_filtered.png"), # filtered latent "assets/examples/plax_latent.mp4", # latent animation "assets/examples/plax_decoded.mp4", # decoded animation ], ], inputs=[ mask_input, class_selection, sampling_steps, ef_slider, animation_steps, cfg_slider, latent_image_display, decoded_image_display, privacy_status, filtered_latent_display, latent_animation_display, decoded_animation_display, ], fn=load_example, label="Click on an example to see the results immediately.", examples_per_page=3, ) # Main container with 4 columns with gr.Row(): # Column 1: Latent Image Generation with gr.Column(): gr.Markdown( '' ) gr.Markdown("### Latent Image Generation") with gr.Row(): # Input mask (binary image) with gr.Column(scale=1): gr.Markdown("Draw the LV mask (white = region of interest)") # Create a black background for the canvas black_background = np.zeros((400, 400), dtype=np.uint8) # Load the default mask image if it exists try: mask_image = Image.open("assets/a4c_seg.png").convert("L") mask_image = mask_image.resize( (400, 400), Image.Resampling.LANCZOS ) # Make it binary (0 or 255) mask_image = ImageOps.autocontrast(mask_image, cutoff=0) mask_image = mask_image.point( lambda p: 255 if p > 127 else 0 ) mask_array = np.array(mask_image) # Create the editor value structure editor_value = { "background": black_background, # Black background "layers": [mask_array], # The mask as an editable layer "composite": mask_array, # The composite image (what's displayed) } except Exception as e: print(f"Error loading mask image: {e}") # Fall back to empty canvas editor_value = black_background # mask_input.value = editor_value mask_input.render() class_selection.render() sampling_steps.render() # Generate button generate_btn = gr.Button("Generate Latent Image", variant="primary") # Display area for latent image (grayscale visualization) latent_image_display.render() # Decode button (initially disabled) decode_btn = gr.Button( "Decode to Pixel Space (Optional)", interactive=False, variant="primary", ) # Display area for decoded image decoded_image_display.render() # Column 2: Privacy Filter with gr.Column(): gr.Markdown( '' ) gr.Markdown("### Privacy Filter") gr.Markdown( "Checks if the generated image is too similar to training data" ) # Privacy check button privacy_btn = gr.Button( "Run Privacy Check", interactive=False, variant="primary" ) # Display area for privacy result status privacy_status.render() # Display area for privacy-filtered latent image filtered_latent_display.render() # Column 3: Animation with gr.Column(): gr.Markdown( '' ) gr.Markdown("### Latent Video Generation") # Ejection Fraction slider ef_slider.render() animation_steps.render() cfg_slider.render() # Animate button animate_btn = gr.Button( "Generate Video", interactive=False, variant="primary" ) # Display area for latent animation (grayscale) latent_animation_display.render() # Column 4: Video Decoding with gr.Column(): gr.Markdown( '' ) gr.Markdown("### Video Decoding") # Decode animation button decode_animation_btn = gr.Button( "Decode Video", interactive=False, variant="primary" ) # Display area for decoded animation decoded_animation_display.render() # Hidden state variables to store the full latent representations latent_image_state = gr.State(None) filtered_latent_state = gr.State(None) latent_animation_state = gr.State(None) # Event handlers class_selection.change( fn=load_view_mask, inputs=[class_selection], outputs=[mask_input], queue=False, ) generate_btn.click( fn=generate_latent_image, inputs=[mask_input, class_selection, sampling_steps], outputs=[latent_image_state], queue=True, ).then( fn=convert_latent_to_display, inputs=[latent_image_state], outputs=[latent_image_display], queue=False, ).then( fn=lambda x: gr.Button( interactive=x is not None ), # Properly update button state inputs=[latent_image_state], outputs=[decode_btn], queue=False, ).then( fn=lambda x: gr.Button( interactive=x is not None ), # Properly update button state inputs=[latent_image_state], outputs=[privacy_btn], queue=False, ) decode_btn.click( fn=decode_latent_to_pixel, inputs=[latent_image_state], outputs=[decoded_image_display], queue=True, ).then( fn=lambda x: gr.Button( interactive=x is not None ), # Properly update button state inputs=[decoded_image_display], outputs=[privacy_btn], queue=False, ) privacy_btn.click( fn=check_privacy, inputs=[latent_image_state, class_selection], outputs=[filtered_latent_state, privacy_status], queue=True, ).then( fn=convert_latent_to_display, inputs=[filtered_latent_state], outputs=[filtered_latent_display], queue=False, ).then( fn=lambda x: gr.Button( interactive=x is not None ), # Properly update button state inputs=[filtered_latent_state], outputs=[animate_btn], queue=False, ) animate_btn.click( fn=generate_animation, inputs=[filtered_latent_state, ef_slider, animation_steps, cfg_slider], outputs=[latent_animation_state], queue=True, ).then( fn=latent_animation_to_grayscale, inputs=[latent_animation_state], outputs=[latent_animation_display], queue=False, ).then( fn=lambda x: gr.Button( interactive=x is not None ), # Properly update button state inputs=[latent_animation_state], outputs=[decode_animation_btn], queue=False, ) decode_animation_btn.click( fn=decode_animation, inputs=[latent_animation_state], # Remove vae_state from inputs outputs=[decoded_animation_display], queue=True, ) return demo if __name__ == "__main__": demo = create_demo() demo.launch()