upload model arch

README.md

@@ -18,7 +18,12 @@ tags:
 # Eagle-2
 
 [\[📂 GitHub\]](https://github.com/NVlabs/EAGLE)   [\[📜 Eagle2 Tech Report\]](http://arxiv.org/abs/2501.14818)
-[\[🗨️ Chat Demo\]](http://eagle-vlm.xyz/)  [\[🤗 HF Demo\]](TODO)  
+[\[🤗 HF Demo\]](https://huggingface.co/spaces/nvidia/Eagle2-Demo)  
+
+# News:
+- We update the model arch to `eagle_2_5_vl` to support  `generate` feature.
+
+
 ## Introduction
 
 We are thrilled to release our latest Eagle2 series Vision-Language Model. Open-source Vision-Language Models (VLMs) have made significant strides in narrowing the gap with proprietary models. However, critical details about data strategies and implementation are often missing, limiting reproducibility and innovation. In this project, we focus on VLM post-training from a data-centric perspective, sharing insights into building effective data strategies from scratch. By combining these strategies with robust training recipes and model design, we introduce Eagle2, a family of performant VLMs. Our work aims to empower the open-source community to develop competitive VLMs with transparent processes.
@@ -66,490 +71,290 @@ We provide the following models:
 
 
 
-We provide a [demo inference script](./demo.py) to help you quickly start using the model. We support different input types: 
+We provide a [inference script](./demo.py) to help you quickly start using the model. We support different input types: 
 - pure text input
 - single image input
 - multiple image input
 - video input
 
-### 0. Install the dependencies
+###  Install the dependencies
 
 ```bash
 pip install transformers
 pip install flash-attn
 ```
-**Note**: Latest version of transformers is not compatible with the model.
 
-### 1. Prepare the Model worker
 
-<details>
-  <summary>Click to expand</summary>
+### single image
 
 ```python
-
-"""
-A model worker executes the model.
-Copied and modified from https://github.com/OpenGVLab/InternVL/blob/main/streamlit_demo/model_worker.py
-"""
-# Importing torch before transformers can cause `segmentation fault`
-from transformers import AutoModel, AutoTokenizer, TextIteratorStreamer, AutoConfig
-
-import argparse
-import base64
-import json
-import os
-import decord
-import threading
-import time
-from io import BytesIO
-from threading import Thread
-import math
+from PIL import Image
 import requests
+from transformers import AutoProcessor, AutoModel
 import torch
-import torchvision.transforms as T
-from PIL import Image
-from torchvision.transforms.functional import InterpolationMode
-import numpy as np
-
-
-IMAGENET_MEAN = (0.485, 0.456, 0.406)
-IMAGENET_STD = (0.229, 0.224, 0.225)
-
-SIGLIP_MEAN = (0.5, 0.5, 0.5)
-SIGLIP_STD = (0.5, 0.5, 0.5)
-
-
-def get_seq_frames(total_num_frames, desired_num_frames=-1, stride=-1):
-    """
-    Calculate the indices of frames to extract from a video.
-
-    Parameters:
-    total_num_frames (int): Total number of frames in the video.
-    desired_num_frames (int): Desired number of frames to extract.
-
-    Returns:
-    list: List of indices of frames to extract.
-    """
-    
-    assert desired_num_frames > 0 or stride > 0 and not (desired_num_frames > 0 and stride > 0)
-
-    if stride > 0:
-        return list(range(0, total_num_frames, stride))
-    
-    # Calculate the size of each segment from which a frame will be extracted
-    seg_size = float(total_num_frames - 1) / desired_num_frames
-
-    seq = []
-    for i in range(desired_num_frames):
-        # Calculate the start and end indices of each segment
-        start = int(np.round(seg_size * i))
-        end = int(np.round(seg_size * (i + 1)))
-
-        # Append the middle index of the segment to the list
-        seq.append((start + end) // 2)
-
-    return seq
-
-def build_video_prompt(meta_list, num_frames, time_position=False):
-    # if time_position is True, the frame_timestamp is used.
-    # 1. pass time_position, 2. use env TIME_POSITION
-    time_position = os.environ.get("TIME_POSITION", time_position)
-    prefix = f"This is a video:\n"
-    for i in range(num_frames):
-        if time_position:
-            frame_txt = f"Frame {i+1} sampled at {meta_list[i]:.2f} seconds: <image>\n"
-        else:
-            frame_txt = f"Frame {i+1}: <image>\n"
-        prefix += frame_txt
-    return prefix
-
-def load_video(video_path, num_frames=64, frame_cache_root=None):
-    if isinstance(video_path, str):
-        video = decord.VideoReader(video_path)
-    elif isinstance(video_path, dict):
-        assert False, 'we not support vidoe: "video_path" as input'
-    fps = video.get_avg_fps()
-    sampled_frames = get_seq_frames(len(video), num_frames)
-    samepld_timestamps = [i / fps for i in sampled_frames]
-    frames = video.get_batch(sampled_frames).asnumpy()
-    images = [Image.fromarray(frame) for frame in frames]
-    
-    return images, build_video_prompt(samepld_timestamps, len(images), time_position=True)
-
-def load_image(image):
-    if isinstance(image, str) and os.path.exists(image):
-        return Image.open(image)
-    elif isinstance(image, dict):
-        if 'disk_path' in image:
-            return Image.open(image['disk_path'])
-        elif 'base64' in image:
-            return Image.open(BytesIO(base64.b64decode(image['base64'])))
-        elif 'url' in image:
-            response = requests.get(image['url'])
-            return Image.open(BytesIO(response.content))
-        elif 'bytes' in image:
-            return Image.open(BytesIO(image['bytes']))
-        else:
-            raise ValueError(f'Invalid image: {image}')
-    else:
-        raise ValueError(f'Invalid image: {image}')
-
-def build_transform(input_size, norm_type='imagenet'):
-    if norm_type == 'imagenet':
-        MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
-    elif norm_type == 'siglip':
-        MEAN, STD = SIGLIP_MEAN, SIGLIP_STD
-        
-    transform = T.Compose([
-        T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
-        T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
-        T.ToTensor(),
-        T.Normalize(mean=MEAN, std=STD)
-    ])
-    return transform
-
-
-def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
-    """
-    previous version mainly foucs on ratio.
-    We also consider area ratio here.
-    """
-    best_factor = float('-inf')
-    best_ratio = (1, 1)
-    area = width * height
-    for ratio in target_ratios:
-        target_aspect_ratio = ratio[0] / ratio[1]
-        ratio_diff = abs(aspect_ratio - target_aspect_ratio)
-        area_ratio = (ratio[0]*ratio[1]*image_size*image_size)/ area
-        """
-        new area > 60% of original image area is enough.
-        """
-        factor_based_on_area_n_ratio = min((ratio[0]*ratio[1]*image_size*image_size)/ area, 0.6)* \
-                                     min(target_aspect_ratio/aspect_ratio, aspect_ratio/target_aspect_ratio)
-        
-        if factor_based_on_area_n_ratio > best_factor:
-            best_factor = factor_based_on_area_n_ratio
-            best_ratio = ratio
-        
-    return best_ratio
-
-
-def dynamic_preprocess(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False):
-    orig_width, orig_height = image.size
-    aspect_ratio = orig_width / orig_height
-
-    # calculate the existing image aspect ratio
-    target_ratios = set(
-        (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
-        i * j <= max_num and i * j >= min_num)
-    target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
-
-    # find the closest aspect ratio to the target
-    target_aspect_ratio = find_closest_aspect_ratio(
-        aspect_ratio, target_ratios, orig_width, orig_height, image_size)
-
-    # calculate the target width and height
-    target_width = image_size * target_aspect_ratio[0]
-    target_height = image_size * target_aspect_ratio[1]
-    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
-
-    # resize the image
-    resized_img = image.resize((target_width, target_height))
-    processed_images = []
-    for i in range(blocks):
-        box = (
-            (i % (target_width // image_size)) * image_size,
-            (i // (target_width // image_size)) * image_size,
-            ((i % (target_width // image_size)) + 1) * image_size,
-            ((i // (target_width // image_size)) + 1) * image_size
-        )
-        # split the image
-        split_img = resized_img.crop(box)
-        processed_images.append(split_img)
-    assert len(processed_images) == blocks
-    if use_thumbnail and len(processed_images) != 1:
-        thumbnail_img = image.resize((image_size, image_size))
-        processed_images.append(thumbnail_img)
-    return processed_images
-
-def split_model(model_path, device):
-
-    device_map = {}
-    world_size = torch.cuda.device_count()
-    config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
-    num_layers = config.llm_config.num_hidden_layers
-
-    print('world_size', world_size)
-    num_layers_per_gpu_ = math.floor(num_layers / (world_size - 1))
-    num_layers_per_gpu = [num_layers_per_gpu_] * world_size
-    num_layers_per_gpu[device] = num_layers - num_layers_per_gpu_ * (world_size-1)
-    print(num_layers_per_gpu)
-    layer_cnt = 0
-    for i, num_layer in enumerate(num_layers_per_gpu):
-        for j in range(num_layer):
-            device_map[f'language_model.model.layers.{layer_cnt}'] = i
-            layer_cnt += 1
-    device_map['vision_model'] = device
-    device_map['mlp1'] = device
-    device_map['language_model.model.tok_embeddings'] = device
-    device_map['language_model.model.embed_tokens'] = device
-    device_map['language_model.output'] = device
-    device_map['language_model.model.norm'] = device
-    device_map['language_model.lm_head'] = device
-    device_map['language_model.model.rotary_emb'] = device
-    device_map[f'language_model.model.layers.{num_layers - 1}'] = device
-    return device_map
-
-class ModelWorker:
-    def __init__(self, model_path, model_name,
-                 load_8bit, device):
-
-        if model_path.endswith('/'):
-            model_path = model_path[:-1]
-        if model_name is None:
-            model_paths = model_path.split('/')
-            if model_paths[-1].startswith('checkpoint-'):
-                self.model_name = model_paths[-2] + '_' + model_paths[-1]
-            else:
-                self.model_name = model_paths[-1]
-        else:
-            self.model_name = model_name
-
-        print(f'Loading the model {self.model_name}')
-
-        tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=False)
-        tokens_to_keep = ['<box>', '</box>', '<ref>', '</ref>']
-        tokenizer.additional_special_tokens = [item for item in tokenizer.additional_special_tokens if item not in tokens_to_keep]
-        self.tokenizer = tokenizer
-        config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
-        model_type = config.vision_config.model_type
-        self.device = torch.cuda.current_device()
-        if model_type == 'siglip_vision_model':
-            self.norm_type = 'siglip'
-        elif model_type == 'MOB':
-            self.norm_type = 'siglip'
-        else:
-            self.norm_type = 'imagenet'
-
-        if any(x in model_path.lower() for x in ['34b']):
-            device_map = split_model(model_path, self.device)
-        else:
-            device_map = None
-        
-        if device_map is not None:    
-            self.model = AutoModel.from_pretrained(model_path, torch_dtype=torch.bfloat16,
-                                               low_cpu_mem_usage=True,
-                                               device_map=device_map, 
-                                               trust_remote_code=True,
-                                               load_in_8bit=load_8bit).eval()
-        else:
-            self.model = AutoModel.from_pretrained(model_path, torch_dtype=torch.bfloat16,
-                                               trust_remote_code=True,
-                                               load_in_8bit=load_8bit).eval()  
-
-        if not load_8bit and device_map is None:
-            self.model = self.model.to(device)
-        self.load_8bit = load_8bit
-        
-        self.model_path = model_path
-        self.image_size = self.model.config.force_image_size
-        self.context_len = tokenizer.model_max_length
-        self.per_tile_len = 256
-
-    def reload_model(self):
-        del self.model
-        torch.cuda.empty_cache()
-        if self.device == 'auto':
-            os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
-            # This can make distributed deployment work properly
-            self.model = AutoModel.from_pretrained(
-                self.model_path,
-                load_in_8bit=self.load_8bit,
-                torch_dtype=torch.bfloat16,
-                device_map=self.device_map,
-                trust_remote_code=True).eval()
-        else:
-            self.model = AutoModel.from_pretrained(
-                self.model_path,
-                load_in_8bit=self.load_8bit,
-                torch_dtype=torch.bfloat16,
-                trust_remote_code=True).eval()
-        if not self.load_8bit and not self.device == 'auto':
-            self.model = self.model.cuda()
-
-    @torch.inference_mode()
-    def generate(self, params):
-        system_message = params['prompt'][0]['content']
-        send_messages = params['prompt'][1:]
-        max_input_tiles = params['max_input_tiles']
-        temperature = params['temperature']
-        top_p = params['top_p']
-        max_new_tokens = params['max_new_tokens']
-        repetition_penalty = params['repetition_penalty']
-        video_frame_num = params.get('video_frame_num', 64)
-        do_sample = True if temperature > 0.0 else False
-
-        global_image_cnt = 0
-        history, pil_images, max_input_tile_list = [], [], []
-        for message in send_messages:
-            if message['role'] == 'user':
-                prefix = ''
-                if 'image' in message:
-                    for image_data in message['image']:
-                        pil_images.append(load_image(image_data))
-                        prefix = prefix + f'<image {global_image_cnt + 1}><image>\n'
-                        global_image_cnt += 1
-                        max_input_tile_list.append(max_input_tiles)
-                if 'video' in message:
-                    for video_data in message['video']:
-                        video_frames, tmp_prefix = load_video(video_data, num_frames=video_frame_num)
-                        pil_images.extend(video_frames)
-                        prefix = prefix + tmp_prefix
-                        global_image_cnt += len(video_frames)
-                        max_input_tile_list.extend([1] * len(video_frames))
-                content = prefix + message['content']
-                history.append([content, ])
-            else:
-                history[-1].append(message['content'])
-        question, history = history[-1][0], history[:-1]
-
-        if global_image_cnt == 1:
-            question = question.replace('<image 1><image>\n', '<image>\n')
-            history = [[item[0].replace('<image 1><image>\n', '<image>\n'), item[1]] for item in history]
-
-
-        try:
-            assert len(max_input_tile_list) == len(pil_images), 'The number of max_input_tile_list and pil_images should be the same.'
-        except Exception as e:
-            from IPython import embed; embed()
-            exit()
-            print(f'Error: {e}')
-            print(f'max_input_tile_list: {max_input_tile_list}, pil_images: {pil_images}')
-            # raise e
-
-        old_system_message = self.model.system_message
-        self.model.system_message = system_message
-        
-        transform = build_transform(input_size=self.image_size, norm_type=self.norm_type)
-        if len(pil_images) > 0:
-            max_input_tiles_limited_by_contect = params['max_input_tiles']
-            while True:
-                image_tiles = []
-                for current_max_input_tiles, pil_image in zip(max_input_tile_list, pil_images):
-                    if self.model.config.dynamic_image_size:
-                        tiles = dynamic_preprocess(
-                            pil_image, image_size=self.image_size, max_num=min(current_max_input_tiles, max_input_tiles_limited_by_contect),
-                            use_thumbnail=self.model.config.use_thumbnail)
-                    else:
-                        tiles = [pil_image]
-                    image_tiles += tiles
-                if (len(image_tiles) * self.per_tile_len < self.context_len):
-                    break
-                else:
-                    max_input_tiles_limited_by_contect -= 2
-                
-                if max_input_tiles_limited_by_contect < 1:
-                    break
-                    
-            pixel_values = [transform(item) for item in image_tiles]
-            pixel_values = torch.stack(pixel_values).to(self.model.device, dtype=torch.bfloat16)
-            print(f'Split images to {pixel_values.shape}')
-        else:
-            pixel_values = None
-
-        generation_config = dict(
-            num_beams=1,
-            max_new_tokens=max_new_tokens,
-            do_sample=do_sample,
-            temperature=temperature,
-            repetition_penalty=repetition_penalty,
-            max_length=self.context_len,
-            top_p=top_p,
-        )
-
-        response = self.model.chat(
-            tokenizer=self.tokenizer,
-            pixel_values=pixel_values,
-            question=question,
-            history=history,
-            return_history=False,
-            generation_config=generation_config,
-        )
-        self.model.system_message = old_system_message
-        return {'text': response, 'error_code': 0}
-
-
-
-
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--model-path', type=str, default='nvidia/Eagle2-2B')
-    parser.add_argument('--model-name', type=str, default='Eagle2-2B')
-    parser.add_argument('--device', type=str, default='cuda')
-    parser.add_argument('--load-8bit', action='store_true')
-    args = parser.parse_args()
-    print(f'args: {args}')
-
-    worker = ModelWorker(
-                         args.model_path,
-                         args.model_name,
-                         args.load_8bit,
-                         args.device)
+model = AutoModel.from_pretrained("nvidia/Eagle2-1B",trust_remote_code=True, torch_dtype=torch.bfloat16)
+processor = AutoProcessor.from_pretrained("nvidia/Eagle2-1B", trust_remote_code=True, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "image": "https://www.ilankelman.org/stopsigns/australia.jpg",
+            },
+            {"type": "text", "text": "Describe this image."},
+        ],
+    }
+]
+
+text_list = [processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True
+)]
+image_inputs, video_inputs = processor.process_vision_info(messages)
+inputs = processor(text = text_list, images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True)
+inputs = inputs.to("cuda")
+model = model.to("cuda")
+generated_ids = model.generate(**inputs, max_new_tokens=1024)
+output_text = processor.batch_decode(
+    generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)
+print(output_text)
 ```
-</details>
+
+### stream generation
+
+```python
+from PIL import Image
+import requests
+from transformers import AutoProcessor, AutoModel, AutoTokenizer
+import torch
+
+from transformers import TextIteratorStreamer
+import threading
 
 
-### 2. Prepare the Prompt
+model = AutoModel.from_pretrained("nvidia/Eagle2-1B",trust_remote_code=True, attn_implementation='flash_attention_2', torch_dtype=torch.bfloat16)
+tokenizer = AutoTokenizer.from_pretrained("nvidia/Eagle2-1B", trust_remote_code=True, use_fast=True)
+processor = AutoProcessor.from_pretrained("nvidia/Eagle2-1B", trust_remote_code=True, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "image": "https://www.ilankelman.org/stopsigns/australia.jpg",
+            },
+            {"type": "text", "text": "Describe this image."},
+        ],
+    }
+]
+
+text_list = [processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True
+)]
+image_inputs, video_inputs = processor.process_vision_info(messages)
+inputs = processor(text = text_list, images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True)
+inputs = inputs.to("cuda")
+model = model.to("cuda")
+
+streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+generation_kwargs = dict(
+    **inputs,
+    streamer=streamer,
+    max_new_tokens=1024,
+    do_sample=True,
+    top_p=0.95,
+    temperature=0.8
+)
+thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
+thread.start()
+
+
+for new_text in streamer:
+    print(new_text, end="", flush=True)
+```
+
+### multiple-images 
 
-- Single image input
 ```python
-prompt = [
-        {'role': 'system', 'content': 'You are a helpful assistant.'},
-        {'role': 'user', 'content': 'Describe this image in details.', 
-            'image':[
-                {'url': 'https://www.nvidia.com/content/dam/en-zz/Solutions/about-nvidia/logo-and-brand/01-nvidia-logo-vert-500x200-2c50-d@2x.png'}
-            ],
-        }
-    ]
+from PIL import Image
+import requests
+from transformers import AutoProcessor, AutoModel
+import torch
+model = AutoModel.from_pretrained("nvidia/Eagle2-1B",trust_remote_code=True, torch_dtype=torch.bfloat16)
+processor = AutoProcessor.from_pretrained("nvidia/Eagle2-1B", trust_remote_code=True, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "image": "https://www.ilankelman.org/stopsigns/australia.jpg",
+            },
+            {
+                "type": "image",
+                "image": "https://www.nvidia.com/content/dam/en-zz/Solutions/about-nvidia/logo-and-brand/[email protected]",
+            },
+            {"type": "text", "text": "Describe these two images."},
+        ],
+    }
+]
+
+text_list = [processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True
+)]
+image_inputs, video_inputs = processor.process_vision_info(messages)
+inputs = processor(text = text_list, images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True)
+inputs = inputs.to("cuda")
+model = model.to("cuda")
+generated_ids = model.generate(**inputs, max_new_tokens=1024)
+output_text = processor.batch_decode(
+    generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)
+print(output_text)
 ```
 
-- Multiple image input
+### single video 
+
 ```python
-prompt = [
-        {'role': 'system', 'content': 'You are a helpful assistant.'},
-        {'role': 'user', 'content': 'Describe these two images in details.', 
-            'image':[
-                {'url': 'https://www.nvidia.com/content/dam/en-zz/Solutions/about-nvidia/logo-and-brand/[email protected]'},
-                {'url': 'https://www.nvidia.com/content/dam/en-zz/Solutions/about-nvidia/logo-and-brand/01-nvidia-logo-vert-500x200-2c50-d@2x.png'}
-            ],
-        }
-    ]
+
+from PIL import Image
+import requests
+from transformers import AutoProcessor, AutoModel
+import torch
+model = AutoModel.from_pretrained("nvidia/Eagle2-1B",trust_remote_code=True, torch_dtype=torch.bfloat16)
+processor = AutoProcessor.from_pretrained("nvidia/Eagle2-1B", trust_remote_code=True, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "video",
+                "video": "../Eagle2-8B/space_woaudio.mp4",
+            },
+            {"type": "text", "text": "Describe this video."},
+        ],
+    }
+]
+
+text_list = [processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True
+)]
+image_inputs, video_inputs, video_kwargs = processor.process_vision_info(messages, return_video_kwargs=True)
+
+inputs = processor(text = text_list, images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True, videos_kwargs=video_kwargs)
+inputs = inputs.to("cuda")
+model = model.to("cuda")
+generated_ids = model.generate(**inputs, max_new_tokens=1024)
+output_text = processor.batch_decode(
+    generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)
+print(output_text)
+
 ```
 
-- Video input
+### multieple videos
+
 ```python
-prompt = [
-        {'role': 'system', 'content': 'You are a helpful assistant.'},
-        {'role': 'user', 'content': 'Describe this video in details.', 
-            'video':[
-                'path/to/your/video.mp4'
-            ],
-        }
-    ]
+from PIL import Image
+import requests
+from transformers import AutoProcessor, AutoModel
+import torch
+model = AutoModel.from_pretrained("nvidia/Eagle2-1B",trust_remote_code=True, torch_dtype=torch.bfloat16)
+processor = AutoProcessor.from_pretrained("nvidia/Eagle2-1B", trust_remote_code=True, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "video",
+                "video": "../Eagle2-8B/space_woaudio.mp4",
+                "nframes": 10,
+            },
+            {
+                "type": "video",
+                "video": "../Eagle2-8B/video_ocr.mp4",
+                "nframes": 10,
+            },
+            {"type": "text", "text": "Describe these two videos respectively."},
+        ],
+    }
+]
+
+text_list = [processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True
+)]
+image_inputs, video_inputs, video_kwargs = processor.process_vision_info(messages, return_video_kwargs=True)
+inputs = processor(text = text_list, images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True, videos_kwargs=video_kwargs)
+inputs = inputs.to("cuda")
+model = model.to("cuda")
+generated_ids = model.generate(**inputs, max_new_tokens=1024)
+output_text = processor.batch_decode(
+    generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)
+print(output_text)
 ```
 
-### 3. Generate the response    
+### batch inference
+
 ```python
-params = {
-    'prompt': prompt,
-    'max_input_tiles': 24,
-    'temperature': 0.7,
-    'top_p': 1.0,
-    'max_new_tokens': 4096,
-    'repetition_penalty': 1.0,
+from PIL import Image
+import requests
+from transformers import AutoProcessor, AutoModel
+import torch
+model = AutoModel.from_pretrained("nvidia/Eagle2-1B",trust_remote_code=True, torch_dtype=torch.bfloat16)
+processor = AutoProcessor.from_pretrained("nvidia/Eagle2-1B", trust_remote_code=True, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+messages1 = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "image": "https://www.ilankelman.org/stopsigns/australia.jpg",
+            },
+            {"type": "text", "text": "Describe this image."},
+        ],
+    }
+]
+
+messages2 = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "image": "https://www.nvidia.com/content/dam/en-zz/Solutions/about-nvidia/logo-and-brand/[email protected]",
+            },
+            {"type": "text", "text": "Describe this image."},
+        ],
     }
-worker.generate(params)
+]
+
+text_list = [processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True
+) for messages in [messages1, messages2]]
+image_inputs, video_inputs = processor.process_vision_info([messages1, messages2])
+inputs = processor(text = text_list, images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True)
+inputs = inputs.to("cuda")
+model = model.to("cuda")
+generated_ids = model.generate(**inputs, max_new_tokens=1024)
+output_text = processor.batch_decode(
+    generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+)
+print(output_text)
 ```
 
 ## TODO

chat_template.json

@@ -0,0 +1 @@
+{"chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}<image {{ image_count.value }}>{% endif %}<image-{{ image_count.value }}>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}<video {{ video_count.value }}>{% endif %}<video-{{ video_count.value }}>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"}

config.json

@@ -1,32 +1,33 @@
 {
-    "_commit_hash": null,
+    "_attn_implementation": "flash_attention_2",
+    "_attn_implementation_autoset": false,
     "_name_or_path": "",
     "architectures": [
-        "Eagle2ChatModel"
+        "Eagle2_5_VLForConditionalGeneration"
     ],
     "auto_map": {
-        "AutoConfig": "configuration_eagle_chat.Eagle2ChatConfig",
-        "AutoModel": "modeling_eagle_chat.Eagle2ChatModel",
-        "AutoModelForCausalLM": "modeling_eagle_chat.Eagle2ChatModel"
-    },
+        "AutoConfig": "configuration_eagle2_5_vl.Eagle2_5_VLConfig",
+        "AutoModel": "modeling_eagle2_5_vl.Eagle2_5_VLForConditionalGeneration"
+      },
     "downsample_ratio": 0.5,
     "dynamic_image_size": true,
-    "efficient_loss": true,
     "force_image_size": 448,
-    "keep_aspect_ratio": false,
-    "llm_config": {
+    "image_token_index": 151667,
+    "max_dynamic_tiles": 12,
+    "min_dynamic_tiles": 1,
+    "mlp_checkpoint": false,
+    "model_type": "eagle_2_5_vl",
+    "pad2square": false,
+    "pre_feature_reduction": false,
+    "select_layer": -1,
+    "template": "qwen2-chat",
+    "text_config": {
         "_name_or_path": "./pretrained/Qwen2_5-1_5B-Instruct",
         "add_cross_attention": false,
         "architectures": [
             "Qwen2ForCausalLM"
         ],
         "attention_dropout": 0.0,
-        "attn_implementation": "flash_attention_2",
-        "auto_map": {
-            "AutoConfig": "configuration_qwen2.Qwen2Config",
-            "AutoModel": "modeling_qwen2.Qwen2Model",
-            "AutoModelForCausalLM": "modeling_qwen2.Qwen2ForCausalLM"
-        },
         "bad_words_ids": null,
         "begin_suppress_tokens": null,
         "bos_token_id": 151643,
@@ -102,105 +103,25 @@
         "use_sliding_window": false,
         "vocab_size": 151674
     },
-    "loss_version": "v4",
-    "max_dynamic_patch": 12,
-    "min_dynamic_patch": 1,
-    "mlp_checkpoint": true,
-    "model_type": "eagle_chat",
-    "pad2square": false,
-    "pre_feature_reduction": false,
-    "ps_version": "v2",
-    "select_layer": -1,
-    "template": "qwen2-chat",
+    "tie_word_embeddings": true,
     "torch_dtype": "bfloat16",
-    "transformers_version": null,
+    "transformers_version": "4.51.0",
     "use_backbone_lora": 0,
     "use_llm_lora": 0,
     "use_thumbnail": true,
     "vision_config": {
-        "_name_or_path": "",
-        "add_cross_attention": false,
-        "architectures": [
-            "SiglipVisionModel"
-        ],
         "attention_dropout": 0.0,
-        "auto_map": {
-            "AutoConfig": "configuration_siglip.SiglipVisionConfig",
-            "AutoModel": "modeling_siglip.SiglipVisionModel"
-        },
-        "bad_words_ids": null,
-        "begin_suppress_tokens": null,
-        "bos_token_id": null,
-        "chunk_size_feed_forward": 0,
-        "cross_attention_hidden_size": null,
-        "decoder_start_token_id": null,
-        "diversity_penalty": 0.0,
-        "do_sample": false,
         "drop_path_rate": 0.1,
-        "early_stopping": false,
-        "encoder_no_repeat_ngram_size": 0,
-        "eos_token_id": null,
-        "exponential_decay_length_penalty": null,
-        "finetuning_task": null,
-        "forced_bos_token_id": null,
-        "forced_eos_token_id": null,
         "hidden_act": "gelu_pytorch_tanh",
         "hidden_size": 1152,
-        "id2label": {
-            "0": "LABEL_0",
-            "1": "LABEL_1"
-        },
         "image_size": 448,
         "intermediate_size": 4304,
-        "is_decoder": false,
-        "is_encoder_decoder": false,
-        "label2id": {
-            "LABEL_0": 0,
-            "LABEL_1": 1
-        },
         "layer_norm_eps": 1e-06,
-        "length_penalty": 1.0,
-        "max_length": 20,
-        "min_length": 0,
         "model_type": "siglip_vision_model",
-        "no_repeat_ngram_size": 0,
         "num_attention_heads": 16,
-        "num_beam_groups": 1,
-        "num_beams": 1,
         "num_channels": 3,
         "num_hidden_layers": 27,
-        "num_image_tokens": 1024,
-        "num_return_sequences": 1,
-        "output_attentions": false,
-        "output_hidden_states": false,
-        "output_scores": false,
-        "pad_token_id": null,
         "patch_size": 14,
-        "prefix": null,
-        "problem_type": null,
-        "projection_dim": 2048,
-        "projector_hidden_act": "gelu_fast",
-        "pruned_heads": {},
-        "remove_invalid_values": false,
-        "repetition_penalty": 1.0,
-        "return_dict": true,
-        "return_dict_in_generate": false,
-        "sep_token_id": null,
-        "suppress_tokens": null,
-        "task_specific_params": null,
-        "temperature": 1.0,
-        "tf_legacy_loss": false,
-        "tie_encoder_decoder": false,
-        "tie_word_embeddings": true,
-        "tokenizer_class": null,
-        "top_k": 50,
-        "top_p": 1.0,
-        "torch_dtype": "bfloat16",
-        "torchscript": false,
-        "transformers_version": "4.37.2",
-        "typical_p": 1.0,
-        "use_bfloat16": false,
-        "vision_use_head": false,
-        "_attn_implementation": "flash_attention_2"
-    }
+        "torch_dtype": "bfloat16"
+      }
 }

configuration_eagle_chat.py → configuration_eagle2_5_vl.py

@@ -1,80 +1,88 @@
 # --------------------------------------------------------
-# Eagle2
+# NVIDIA
 # Copyright (c) 2025 NVIDIA
-# Licensed under The Apache License [see LICENSE for details]
+# Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
 
 import copy
 
-from transformers import AutoConfig, LlamaConfig
+from transformers.models.llama.configuration_llama import LlamaConfig
+from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
 from transformers.configuration_utils import PretrainedConfig
 from transformers.utils import logging
 from transformers.models.siglip.configuration_siglip import SiglipVisionConfig
-from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
 logger = logging.get_logger(__name__)
 
-class Eagle2ChatConfig(PretrainedConfig):
-    model_type = 'eagle_chat'
-    is_composition = True
 
+class Eagle2_5_VLConfig(PretrainedConfig):
+    model_type = 'eagle_2_5_vl'
+    is_composition = True
+    sub_configs = {"vision_config": SiglipVisionConfig, "text_config": Qwen2Config}
     def __init__(
             self,
             vision_config=None,
-            llm_config=None,
+            text_config=None,
             use_backbone_lora=0,
             use_llm_lora=0,
-            select_layer=-1,
+            pad2square=False,
+            select_layer=-4,
             force_image_size=None,
             downsample_ratio=0.5,
             template=None,
             dynamic_image_size=False,
             use_thumbnail=False,
-            min_dynamic_patch=1,
-            max_dynamic_patch=6,
-            mlp_checkpoint=True,
-            pre_feature_reduction=False,
-            keep_aspect_ratio=False,
-            vocab_size=-1,
+            loss_version='v1',
+            min_dynamic_tiles=1,
+            max_dynamic_tiles=6,
+            mlp_checkpoint=False,
+            initializer_range=0.02,
+            _attn_implementation='flash_attention_2',
+            _attn_implementation_autoset=False,
+            llm_config=None,
+            image_token_index=None,
             **kwargs):
         super().__init__(**kwargs)
 
         if vision_config is None:
-            vision_config = {}
-            logger.info('vision_config is None. Initializing Vision Encoders with default values.')
+            vision_config = {'model_type': 'siglip_vision_model'}
+            logger.info('vision_config is None. Initializing the InternVisionConfig with default values.')
 
-        if llm_config is None:
-            llm_config = {}
-            logger.info('llm_config is None. Initializing the LLM config with default values')
+        if text_config is None:
+            text_config = {'architectures': ['Qwen2ForCausalLM']}
+            logger.info('text_config is None. Initializing the LlamaConfig config with default values (`LlamaConfig`).')
 
         if vision_config['model_type'] == 'siglip_vision_model':
             self.vision_config = SiglipVisionConfig(**vision_config)
         else:
             raise ValueError('Unsupported model_type: {}'.format(vision_config['model_type']))
 
-        if llm_config['architectures'][0] == 'LlamaForCausalLM':
-            self.llm_config = LlamaConfig(**llm_config)
-        elif llm_config['architectures'][0] == 'Qwen2ForCausalLM':
-            self.llm_config = Qwen2Config(**llm_config)
+
+        if text_config['architectures'][0] == 'LlamaForCausalLM':
+            self.text_config = LlamaConfig(**text_config)
+        elif text_config['architectures'][0] == 'Qwen2ForCausalLM':
+            self.text_config = Qwen2Config(**text_config)
         else:
-            raise ValueError('Unsupported architecture: {}'.format(llm_config['architectures'][0]))
+            raise ValueError('Unsupported architecture: {}'.format(text_config['architectures'][0]))
         self.use_backbone_lora = use_backbone_lora
         self.use_llm_lora = use_llm_lora
+        self.mlp_checkpoint = mlp_checkpoint
+        self.pad2square = pad2square
         self.select_layer = select_layer
         self.force_image_size = force_image_size
         self.downsample_ratio = downsample_ratio
         self.template = template
         self.dynamic_image_size = dynamic_image_size
         self.use_thumbnail = use_thumbnail
-        self.min_dynamic_patch = min_dynamic_patch
-        self.max_dynamic_patch = max_dynamic_patch
-        self.mlp_checkpoint = mlp_checkpoint
-        self.pre_feature_reduction = pre_feature_reduction
-        self.keep_aspect_ratio = keep_aspect_ratio
-        self.vocab_size = self.llm_config.vocab_size
-        logger.info(f'keep_aspect_ratio: {self.keep_aspect_ratio}')
-        logger.info(f'vision_select_layer: {self.select_layer}')
-        logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')
-        logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')
+        self.loss_version = loss_version
+        self.initializer_range = initializer_range
+        self.min_dynamic_tiles = min_dynamic_tiles
+        self.max_dynamic_tiles = max_dynamic_tiles
+        self.tie_word_embeddings = self.text_config.tie_word_embeddings
+        self._attn_implementation = _attn_implementation
+        self._attn_implementation_autoset = _attn_implementation_autoset
+        self.image_token_index = image_token_index
+        logger.info(f'min_dynamic_tiles: {self.min_dynamic_tiles}')
+        logger.info(f'max_dynamic_tiles: {self.max_dynamic_tiles}')
 
     def to_dict(self):
         """
@@ -85,18 +93,21 @@ class Eagle2ChatConfig(PretrainedConfig):
         """
         output = copy.deepcopy(self.__dict__)
         output['vision_config'] = self.vision_config.to_dict()
-        output['llm_config'] = self.llm_config.to_dict()
+        output['text_config'] = self.text_config.to_dict()
         output['model_type'] = self.__class__.model_type
         output['use_backbone_lora'] = self.use_backbone_lora
         output['use_llm_lora'] = self.use_llm_lora
+        output['pad2square'] = self.pad2square
         output['select_layer'] = self.select_layer
         output['force_image_size'] = self.force_image_size
         output['downsample_ratio'] = self.downsample_ratio
         output['template'] = self.template
         output['dynamic_image_size'] = self.dynamic_image_size
         output['use_thumbnail'] = self.use_thumbnail
-        output['min_dynamic_patch'] = self.min_dynamic_patch
-        output['max_dynamic_patch'] = self.max_dynamic_patch
-        output['keep_aspect_ratio'] = self.keep_aspect_ratio
+        output['min_dynamic_tiles'] = self.min_dynamic_tiles
+        output['max_dynamic_tiles'] = self.max_dynamic_tiles
+        output['tie_word_embeddings'] = self.tie_word_embeddings
+        output['_attn_implementation'] = self._attn_implementation
+        output['_attn_implementation_autoset'] = self._attn_implementation_autoset
 
         return output

demo.py

@@ -1,428 +1,55 @@
-
-"""
-A model worker executes the model.
-"""
-from transformers import AutoModel, AutoTokenizer, TextIteratorStreamer, AutoConfig
-import argparse
-import base64
-import json
-import os
-import decord
-import threading
-import time
-from io import BytesIO
-from threading import Thread
-import math
+from PIL import Image
 import requests
+from transformers import AutoProcessor, AutoModel, AutoTokenizer
 import torch
-import torchvision.transforms as T
-from PIL import Image
-from torchvision.transforms.functional import InterpolationMode
-
-import numpy as np
-
-IMAGENET_MEAN = (0.485, 0.456, 0.406)
-IMAGENET_STD = (0.229, 0.224, 0.225)
-
-SIGLIP_MEAN = (0.5, 0.5, 0.5)
-SIGLIP_STD = (0.5, 0.5, 0.5)
-
-
-def get_seq_frames(total_num_frames, desired_num_frames=-1, stride=-1):
-    """
-    Calculate the indices of frames to extract from a video.
-
-    Parameters:
-    total_num_frames (int): Total number of frames in the video.
-    desired_num_frames (int): Desired number of frames to extract.
-
-    Returns:
-    list: List of indices of frames to extract.
-    """
-    
-    assert desired_num_frames > 0 or stride > 0 and not (desired_num_frames > 0 and stride > 0)
-
-    if stride > 0:
-        return list(range(0, total_num_frames, stride))
-    
-    # Calculate the size of each segment from which a frame will be extracted
-    seg_size = float(total_num_frames - 1) / desired_num_frames
-
-    seq = []
-    for i in range(desired_num_frames):
-        # Calculate the start and end indices of each segment
-        start = int(np.round(seg_size * i))
-        end = int(np.round(seg_size * (i + 1)))
-
-        # Append the middle index of the segment to the list
-        seq.append((start + end) // 2)
-
-    return seq
-
-def build_video_prompt(meta_list, num_frames, time_position=False):
-    # if time_position is True, the frame_timestamp is used.
-    # 1. pass time_position, 2. use env TIME_POSITION
-    time_position = os.environ.get("TIME_POSITION", time_position)
-    prefix = f"This is a video:\n"
-    for i in range(num_frames):
-        if time_position:
-            frame_txt = f"Frame {i+1} sampled at {meta_list[i]:.2f} seconds: <image>\n"
-        else:
-            frame_txt = f"Frame {i+1}: <image>\n"
-        prefix += frame_txt
-    return prefix
-
-def load_video(video_path, num_frames=64, frame_cache_root=None):
-    if isinstance(video_path, str):
-        video = decord.VideoReader(video_path)
-    elif isinstance(video_path, dict):
-        assert False, 'we not support vidoe: "video_path" as input'
-    fps = video.get_avg_fps()
-    sampled_frames = get_seq_frames(len(video), num_frames)
-    samepld_timestamps = [i / fps for i in sampled_frames]
-    frames = video.get_batch(sampled_frames).asnumpy()
-    images = [Image.fromarray(frame) for frame in frames]
-    
-    return images, build_video_prompt(samepld_timestamps, len(images), time_position=True)
-
-def load_image(image):
-    if isinstance(image, str) and os.path.exists(image):
-        return Image.open(image)
-    elif isinstance(image, dict):
-        if 'disk_path' in image:
-            return Image.open(image['disk_path'])
-        elif 'base64' in image:
-            return Image.open(BytesIO(base64.b64decode(image['base64'])))
-        elif 'url' in image:
-            response = requests.get(image['url'])
-            return Image.open(BytesIO(response.content))
-        elif 'bytes' in image:
-            return Image.open(BytesIO(image['bytes']))
-        else:
-            raise ValueError(f'Invalid image: {image}')
-    else:
-        raise ValueError(f'Invalid image: {image}')
-
-def build_transform(input_size, norm_type='imagenet'):
-    if norm_type == 'imagenet':
-        MEAN, STD = IMAGENET_MEAN, IMAGENET_STD
-    elif norm_type == 'siglip':
-        MEAN, STD = SIGLIP_MEAN, SIGLIP_STD
-        
-    transform = T.Compose([
-        T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),
-        T.Resize((input_size, input_size), interpolation=InterpolationMode.BICUBIC),
-        T.ToTensor(),
-        T.Normalize(mean=MEAN, std=STD)
-    ])
-    return transform
-
-
-def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
-    """
-    previous version mainly foucs on ratio.
-    We also consider area ratio here.
-    """
-    best_factor = float('-inf')
-    best_ratio = (1, 1)
-    area = width * height
-    for ratio in target_ratios:
-        target_aspect_ratio = ratio[0] / ratio[1]
-        ratio_diff = abs(aspect_ratio - target_aspect_ratio)
-        area_ratio = (ratio[0]*ratio[1]*image_size*image_size)/ area
-        """
-        new area > 60% of original image area is enough.
-        """
-        factor_based_on_area_n_ratio = min((ratio[0]*ratio[1]*image_size*image_size)/ area, 0.6)* \
-                                     min(target_aspect_ratio/aspect_ratio, aspect_ratio/target_aspect_ratio)
-        
-        if factor_based_on_area_n_ratio > best_factor:
-            best_factor = factor_based_on_area_n_ratio
-            best_ratio = ratio
-        
-    return best_ratio
-
-
-def dynamic_preprocess(image, min_num=1, max_num=6, image_size=448, use_thumbnail=False):
-    orig_width, orig_height = image.size
-    aspect_ratio = orig_width / orig_height
-
-    # calculate the existing image aspect ratio
-    target_ratios = set(
-        (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
-        i * j <= max_num and i * j >= min_num)
-    target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
-
-    # find the closest aspect ratio to the target
-    target_aspect_ratio = find_closest_aspect_ratio(
-        aspect_ratio, target_ratios, orig_width, orig_height, image_size)
-
-    # calculate the target width and height
-    target_width = image_size * target_aspect_ratio[0]
-    target_height = image_size * target_aspect_ratio[1]
-    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
-
-    # resize the image
-    resized_img = image.resize((target_width, target_height))
-    processed_images = []
-    for i in range(blocks):
-        box = (
-            (i % (target_width // image_size)) * image_size,
-            (i // (target_width // image_size)) * image_size,
-            ((i % (target_width // image_size)) + 1) * image_size,
-            ((i // (target_width // image_size)) + 1) * image_size
-        )
-        # split the image
-        split_img = resized_img.crop(box)
-        processed_images.append(split_img)
-    assert len(processed_images) == blocks
-    if use_thumbnail and len(processed_images) != 1:
-        thumbnail_img = image.resize((image_size, image_size))
-        processed_images.append(thumbnail_img)
-    return processed_images
-
-def split_model(model_path, device):
-
-    device_map = {}
-    world_size = torch.cuda.device_count()
-    config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
-    num_layers = config.llm_config.num_hidden_layers
-
-    num_layers_per_gpu_ = math.floor(num_layers / (world_size - 1))
-    num_layers_per_gpu = [num_layers_per_gpu_] * world_size
-    num_layers_per_gpu[device] = num_layers - num_layers_per_gpu_ * (world_size-1)
-    layer_cnt = 0
-    for i, num_layer in enumerate(num_layers_per_gpu):
-        for j in range(num_layer):
-            device_map[f'language_model.model.layers.{layer_cnt}'] = i
-            layer_cnt += 1
-    device_map['vision_model'] = device
-    device_map['mlp1'] = device
-    device_map['language_model.model.tok_embeddings'] = device
-    device_map['language_model.model.embed_tokens'] = device
-    device_map['language_model.output'] = device
-    device_map['language_model.model.norm'] = device
-    device_map['language_model.lm_head'] = device
-    device_map['language_model.model.rotary_emb'] = device
-    device_map[f'language_model.model.layers.{num_layers - 1}'] = device
-    return device_map
-
-class ModelWorker:
-    def __init__(self, model_path, model_name,
-                 load_8bit, device):
-
-        if model_path.endswith('/'):
-            model_path = model_path[:-1]
-        if model_name is None:
-            model_paths = model_path.split('/')
-            if model_paths[-1].startswith('checkpoint-'):
-                self.model_name = model_paths[-2] + '_' + model_paths[-1]
-            else:
-                self.model_name = model_paths[-1]
-        else:
-            self.model_name = model_name
-
-        print(f'Loading the model {self.model_name}')
-
-        tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=False)
-        tokens_to_keep = ['<box>', '</box>', '<ref>', '</ref>']
-        tokenizer.additional_special_tokens = [item for item in tokenizer.additional_special_tokens if item not in tokens_to_keep]
-        self.tokenizer = tokenizer
-        config = AutoConfig.from_pretrained(model_path, trust_remote_code=True)
-        model_type = config.vision_config.model_type
-        self.device = torch.cuda.current_device()
-        if model_type == 'siglip_vision_model':
-            self.norm_type = 'siglip'
-        elif model_type == 'MOB':
-            self.norm_type = 'siglip'
-        else:
-            self.norm_type = 'imagenet'
-        print('norm_type: ', self.norm_type)
-        if any(x in model_path.lower() for x in ['34b']):
-            device_map = split_model(model_path, self.device)
-        else:
-            device_map = None
-
-        if device_map is not None:    
-            self.model = AutoModel.from_pretrained(model_path, torch_dtype=torch.bfloat16,
-                                               low_cpu_mem_usage=True,
-                                               device_map=device_map, 
-                                               trust_remote_code=True,
-                                               load_in_8bit=load_8bit).eval()
-        else:
-            self.model = AutoModel.from_pretrained(model_path, torch_dtype=torch.bfloat16,
-                                               trust_remote_code=True,
-                                               load_in_8bit=load_8bit).eval()  
-        if not load_8bit and device_map is None:
-            self.model = self.model.to(device)
-        self.load_8bit = load_8bit
-        
-        self.model_path = model_path
-        self.image_size = self.model.config.force_image_size
-        self.context_len = tokenizer.model_max_length
-        self.per_tile_len = 256
-        print(self.model)
-    def reload_model(self):
-        del self.model
-        torch.cuda.empty_cache()
-        if self.device == 'auto':
-            os.environ['CUDA_LAUNCH_BLOCKING'] = '1'
-            # This can make distributed deployment work properly
-            self.model = AutoModel.from_pretrained(
-                self.model_path,
-                load_in_8bit=self.load_8bit,
-                torch_dtype=torch.bfloat16,
-                device_map=self.device_map,
-                trust_remote_code=True).eval()
-        else:
-            self.model = AutoModel.from_pretrained(
-                self.model_path,
-                load_in_8bit=self.load_8bit,
-                torch_dtype=torch.bfloat16,
-                trust_remote_code=True).eval()
-        if not self.load_8bit and not self.device == 'auto':
-            self.model = self.model.cuda()
-
-    @torch.inference_mode()
-    def generate(self, params):
-        system_message = params['prompt'][0]['content']
-        send_messages = params['prompt'][1:]
-        max_input_tiles = params['max_input_tiles']
-        temperature = params['temperature']
-        top_p = params['top_p']
-        max_new_tokens = params['max_new_tokens']
-        repetition_penalty = params['repetition_penalty']
-        video_frame_num = params.get('video_frame_num', 64)
-        do_sample = True if temperature > 0.0 else False
-
-        global_image_cnt = 0
-        history, pil_images, max_input_tile_list = [], [], []
-        
-        for message in send_messages:
-            if message['role'] == 'user':
-                prefix = ''
-                if 'image' in message:
-                    for image_data in message['image']:
-                        pil_images.append(load_image(image_data))
-                        prefix = prefix + f'<image {global_image_cnt + 1}><image>\n'
-                        global_image_cnt += 1
-                        max_input_tile_list.append(max_input_tiles)
-                if 'video' in message:
-                    for video_data in message['video']:
-                        video_frames, tmp_prefix = load_video(video_data, num_frames=video_frame_num)
-                        pil_images.extend(video_frames)
-                        prefix = prefix + tmp_prefix
-                        global_image_cnt += len(video_frames)
-                        max_input_tile_list.extend([1] * len(video_frames))
-                content = prefix + message['content']
-                history.append([content, ])
-            else:
-                history[-1].append(message['content'])
-        question, history = history[-1][0], history[:-1]
-
-        if global_image_cnt == 1:
-            question = question.replace('<image 1><image>\n', '<image>\n')
-            history = [[item[0].replace('<image 1><image>\n', '<image>\n'), item[1]] for item in history]
-
-
-        try:
-            assert len(max_input_tile_list) == len(pil_images), 'The number of max_input_tile_list and pil_images should be the same.'
-        except Exception as e:
-            from IPython import embed; embed()
-            exit()
-            print(f'Error: {e}')
-            print(f'max_input_tile_list: {max_input_tile_list}, pil_images: {pil_images}')
-            # raise e
-
-        old_system_message = self.model.system_message
-        self.model.system_message = system_message
-        
-        transform = build_transform(input_size=self.image_size, norm_type=self.norm_type)
-        if len(pil_images) > 0:
-            max_input_tiles_limited_by_contect = params['max_input_tiles']
-            while True:
-                image_tiles = []
-                num_patches_list = []
-                for current_max_input_tiles, pil_image in zip(max_input_tile_list, pil_images):
-                    if self.model.config.dynamic_image_size:
-                        tiles = dynamic_preprocess(
-                            pil_image, image_size=self.image_size, max_num=min(current_max_input_tiles, max_input_tiles_limited_by_contect),
-                            use_thumbnail=self.model.config.use_thumbnail)
-                    else:
-                        tiles = [pil_image]
-                    num_patches_list.append(len(tiles))
-                    image_tiles += tiles
-                if (len(image_tiles) * self.per_tile_len < self.context_len):
-                    break
-                else:
-                    max_input_tiles_limited_by_contect -= 2
-                
-                if max_input_tiles_limited_by_contect < 1:
-                    break
-                    
-            pixel_values = [transform(item) for item in image_tiles]
-
-            
-            pixel_values = torch.stack(pixel_values).to(self.model.device, dtype=torch.bfloat16)
-            
-        else:
-            pixel_values = None
-
-        generation_config = dict(
-            num_beams=1,
-            max_new_tokens=max_new_tokens,
-            do_sample=do_sample,
-            temperature=temperature,
-            repetition_penalty=repetition_penalty,
-            max_length=self.context_len,
-            top_p=top_p,
-        )
-        print(f'pixel_values: {pixel_values.shape}')
-        response = self.model.chat(
-            tokenizer=self.tokenizer,
-            pixel_values=pixel_values,
-            question=question,
-            history=history,
-            return_history=False,
-            num_patches_list=num_patches_list,
-            generation_config=generation_config,
-        )
-        self.model.system_message = old_system_message
-        return {'text': response, 'error_code': 0}
-
-
-
 
+from transformers import TextIteratorStreamer
+import threading
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser()
-    parser.add_argument('--model-path', type=str, default='/home/zhidingy/workspace/eagle-next/internvl_chat/work_dirs/release/test/Eagle2-2B')
-    parser.add_argument('--model-name', type=str, default='Eagle2')
-    parser.add_argument('--device', type=str, default='cuda')
-    parser.add_argument('--load-8bit', action='store_true')
-    args = parser.parse_args()
-    print(f'args: {args}')
 
-    worker = ModelWorker(
-                         args.model_path,
-                         args.model_name,
-                         args.load_8bit,
-                         args.device)
-    prompt = [
-        {'role': 'system', 'content': 'You are a helpful assistant.'},
-        {'role': 'user', 'content': 'Describe these two images in details respectively.', 
-            'image':[
-                {'url': 'https://www.nvidia.com/content/dam/en-zz/Solutions/about-nvidia/logo-and-brand/[email protected]'},
-                {'url': "https://www.google.com.hk/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png"}
-            ]
-        }
-    ]
-    params = {
-        'prompt': prompt,
-        'max_input_tiles': 24,
-        'temperature': 0.7,
-        'top_p': 1.0,
-        'max_new_tokens': 4096,
-        'repetition_penalty': 1.0,
+model = AutoModel.from_pretrained("/home/zhidingy/workspace/eagle-next/internvl_chat/work_dirs/release/test/Eagle2-2B",trust_remote_code=True, attn_implementation='flash_attention_2', torch_dtype=torch.bfloat16)
+tokenizer = AutoTokenizer.from_pretrained("/home/zhidingy/workspace/eagle-next/internvl_chat/work_dirs/release/test/Eagle2-2B", trust_remote_code=True, use_fast=True)
+processor = AutoProcessor.from_pretrained("/home/zhidingy/workspace/eagle-next/internvl_chat/work_dirs/release/test/Eagle2-2B", trust_remote_code=True, use_fast=True)
+processor.tokenizer.padding_side = "left"
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image",
+                "image": "https://www.ilankelman.org/stopsigns/australia.jpg",
+            },
+            {"type": "text", "text": "Describe this image."},
+        ],
     }
-    print(worker.generate(params))
+]
+
+text_list = [processor.apply_chat_template(
+    messages, tokenize=False, add_generation_prompt=True
+)]
+image_inputs, video_inputs = processor.process_vision_info(messages)
+inputs = processor(text = text_list, images=image_inputs, videos=video_inputs, return_tensors="pt", padding=True)
+inputs = inputs.to("cuda")
+model = model.to("cuda")
+# generated_ids = model.generate(**inputs, max_new_tokens=1024)
+# output_text = processor.batch_decode(
+#     generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False
+# )
+# print(output_text)
+
+streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+generation_kwargs = dict(
+    **inputs,
+    streamer=streamer,
+    max_new_tokens=1024,
+    do_sample=True,
+    top_p=0.95,
+    temperature=0.8
+)
+thread = threading.Thread(target=model.generate, kwargs=generation_kwargs)
+thread.start()
+
+for new_text in streamer:
+    print(new_text, end="", flush=True)

image_processing_eagle2.py

@@ -0,0 +1,715 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for LLaVa-Onevision."""
+
+import math
+from typing import Dict, Iterable, List, Optional, Tuple, Union
+
+import numpy as np
+
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict, select_best_resolution
+from transformers.image_transforms import (
+    PaddingMode,
+    convert_to_rgb,
+    pad,
+    resize,
+    to_channel_dimension_format,
+)
+from transformers.image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    IMAGENET_STANDARD_MEAN, # 0.5, 0.5, 0.5
+    IMAGENET_STANDARD_STD, # 0.5, 0.5, 0.5
+    ChannelDimension,
+    ImageInput,
+    PILImageResampling,
+    get_image_size,
+    infer_channel_dimension_format,
+    is_scaled_image,
+    make_flat_list_of_images,
+    to_numpy_array,
+    valid_images,
+    validate_preprocess_arguments,
+)
+from transformers.utils import TensorType, is_vision_available, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+if is_vision_available():
+    from PIL import Image
+
+def crop(img: np.ndarray, left: int, top: int, right: int, bottom: int, input_data_format: ChannelDimension) -> np.ndarray:
+    """Crop the given numpy array.
+    
+    Args:
+        img (np.ndarray): Image to be cropped. Format should be (H, W, C) or (H, W).
+        left (int): The left coordinate of the crop box.
+        top (int): The top coordinate of the crop box.
+        right (int): The right coordinate of the crop box.
+        bottom (int): The bottom coordinate of the crop box.
+        
+    Returns:
+        np.ndarray: Cropped image.
+    """
+    if not isinstance(img, np.ndarray):
+        raise TypeError('img should be numpy array. Got {}'.format(type(img)))
+    
+    if img.ndim not in [2, 3]:
+        raise ValueError('Image should have 2 or 3 dimensions. Got {}'.format(img.ndim))
+    
+    if input_data_format == ChannelDimension.LAST:
+        img_height = img.shape[0]
+        img_width = img.shape[1]
+    else:
+        img_height = img.shape[1]
+        img_width = img.shape[2]
+    
+    if top < 0 or left < 0 or bottom > img_height or right > img_width:
+        raise ValueError('Crop coordinates out of bounds')
+    
+    if top >= bottom or left >= right:
+        raise ValueError('Invalid crop coordinates')
+    if input_data_format == ChannelDimension.LAST:
+        return img[top:bottom, left:right, :]
+    else:
+        return img[:, top:bottom, left:right]
+
+# Copied from transformers.models.llava_next.image_processing_llava_next.divide_to_patches
+def divide_to_patches(image: np.array, patch_size: int, input_data_format) -> List[np.array]:
+    """
+    Divides an image into patches of a specified size.
+
+    Args:
+        image (`np.array`):
+            The input image.
+        patch_size (`int`):
+            The size of each patch.
+        input_data_format (`ChannelDimension` or `str`):
+            The channel dimension format of the input image.
+
+    Returns:
+        list: A list of np.array representing the patches.
+    """
+    patches = []
+    height, width = get_image_size(image, channel_dim=input_data_format)
+    for i in range(0, height, patch_size):
+        for j in range(0, width, patch_size):
+            if input_data_format == ChannelDimension.LAST:
+                patch = image[i : i + patch_size, j : j + patch_size]
+            else:
+                patch = image[:, i : i + patch_size, j : j + patch_size]
+            patches.append(patch)
+
+    return patches
+
+
+# Copied from transformers.models.llava_next.image_processing_llava_next.expand_to_square
+def expand_to_square(image: np.array, background_color, input_data_format) -> np.array:
+    """
+    Expands an image to a square by adding a background color.
+    """
+
+    height, width = get_image_size(image, channel_dim=input_data_format)
+    if width == height:
+        return image
+    elif width > height:
+        result = np.ones((width, width, image.shape[2]), dtype=image.dtype) * background_color
+        result[(width - height) // 2 : (width - height) // 2 + height, :] = image
+        return result
+    else:
+        result = np.ones((height, height, image.shape[2]), dtype=image.dtype) * background_color
+        result[:, (height - width) // 2 : (height - width) // 2 + width] = image
+        return result
+
+
+# Copied from transformers.models.llava_next.image_processing_llava_next._get_patch_output_size
+def _get_patch_output_size(image, target_resolution, input_data_format):
+    original_height, original_width = get_image_size(image, channel_dim=input_data_format)
+    target_height, target_width = target_resolution
+
+    scale_w = target_width / original_width
+    scale_h = target_height / original_height
+
+    if scale_w < scale_h:
+        new_width = target_width
+        new_height = min(math.ceil(original_height * scale_w), target_height)
+    else:
+        new_height = target_height
+        new_width = min(math.ceil(original_width * scale_h), target_width)
+
+    return new_height, new_width
+
+
+class Eagle2ImageProcessor(BaseImageProcessor):
+    r"""
+    Constructs a LLaVa-Onevision image processor. Based on [`SiglipImageProcessor`] with incorporation of processing each video frame.
+
+    Args:
+        do_resize (`bool`, *optional*, defaults to `True`):
+            Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by
+            `do_resize` in the `preprocess` method.
+        size (`Dict[str, int]` *optional*, defaults to `{"shortest_edge": 224}`):
+            Size of the image after resizing. The shortest edge of the image is resized to size["shortest_edge"], with
+            the longest edge resized to keep the input aspect ratio. Can be overridden by `size` in the `preprocess`
+            method.
+        image_grid_pinpoints (`List` *optional*, defaults to `[[672, 336], [336, 672], [672, 672], [336, 1008], [1008, 336]]`):
+            A list of possible resolutions to use for processing high resolution images. The best resolution is selected
+            based on the original size of the image. Can be overridden by `image_grid_pinpoints` in the `preprocess`
+            method. Not used for processinf videos.
+        resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
+            Resampling filter to use if resizing the image. Can be overridden by `resample` in the `preprocess` method.
+        do_rescale (`bool`, *optional*, defaults to `True`):
+            Whether to rescale the image by the specified scale `rescale_factor`. Can be overridden by `do_rescale` in
+            the `preprocess` method.
+        rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
+            Scale factor to use if rescaling the image. Can be overridden by `rescale_factor` in the `preprocess`
+            method.
+        do_normalize (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the image. Can be overridden by `do_normalize` in the `preprocess` method.
+        image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
+            Mean to use if normalizing the image. This is a float or list of floats the length of the number of
+            channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method.
+        image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
+            Standard deviation to use if normalizing the image. This is a float or list of floats the length of the
+            number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method.
+            Can be overridden by the `image_std` parameter in the `preprocess` method.
+        do_pad (`bool`, *optional*, defaults to `True`):
+                Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest
+                number of patches in the batch. Padding will be applied to the bottom and right with zeros.
+        do_convert_rgb (`bool`, *optional*, defaults to `True`):
+            Whether to convert the image to RGB.
+    """
+
+    model_input_names = ["pixel_values_videos"]
+
+    def __init__(
+        self,
+        do_resize: bool = True,
+        size: Dict[str, int] = None,
+        resample: PILImageResampling = PILImageResampling.BICUBIC,
+        do_rescale: bool = True,
+        rescale_factor: Union[int, float] = 1 / 255,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_pad: Optional[bool] = True,
+        do_convert_rgb: bool = True,
+        min_dynamic_tiles: int = 1,
+        max_dynamic_tiles: int = 12,
+        use_thumbnail: bool = True,
+        pad_during_tiling: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        size = size if size is not None else {"height": 384, "width": 384}
+        size = get_size_dict(size, default_to_square=False)
+
+        self.do_resize = do_resize
+        self.size = size
+        self.resample = resample
+        self.do_rescale = do_rescale
+        self.rescale_factor = rescale_factor
+        self.do_normalize = do_normalize
+        self.image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
+        self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD
+        self.do_pad = do_pad
+        self.do_convert_rgb = do_convert_rgb
+        self.min_dynamic_tiles = min_dynamic_tiles
+        self.max_dynamic_tiles = max_dynamic_tiles
+        self.use_thumbnail = use_thumbnail
+        self.pad_during_tiling = pad_during_tiling
+        
+    # Copied from transformers.models.llava_next.image_processing_llava_next.LlavaNextImageProcessor.pad
+    def pad(
+        self,
+        image: np.ndarray,
+        padding: Union[int, Tuple[int, int], Iterable[Tuple[int, int]]],
+        mode: PaddingMode = PaddingMode.CONSTANT,
+        constant_values: Union[float, Iterable[float]] = 0.0,
+        data_format: Optional[Union[str, ChannelDimension]] = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ) -> np.ndarray:
+        """
+        Pads the `image` with the specified `padding` and `mode`. Padding can be in the (`height`, `width`)
+        dimension of in the (`num_patches`) dimension. In the second case an iterable if tuples is expected
+        as input.
+
+        Args:
+            image (`np.ndarray`):
+                The image to pad.
+            padding (`int` or `Tuple[int, int]` or `Iterable[Tuple[int, int]]`):
+                Padding to apply to the edges of the height, width axes. Can be one of three formats:
+                - `((before_height, after_height), (before_width, after_width))` unique pad widths for each axis.
+                - `((before, after),)` yields same before and after pad for height and width.
+                - `(pad,)` or int is a shortcut for before = after = pad width for all axes.
+            mode (`PaddingMode`):
+                The padding mode to use. Can be one of:
+                    - `"constant"`: pads with a constant value.
+                    - `"reflect"`: pads with the reflection of the vector mirrored on the first and last values of the
+                    vector along each axis.
+                    - `"replicate"`: pads with the replication of the last value on the edge of the array along each axis.
+                    - `"symmetric"`: pads with the reflection of the vector mirrored along the edge of the array.
+            constant_values (`float` or `Iterable[float]`, *optional*):
+                The value to use for the padding if `mode` is `"constant"`.
+            data_format (`str` or `ChannelDimension`, *optional*):
+                The channel dimension format for the output image. Can be one of:
+                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                If unset, will use same as the input image.
+            input_data_format (`str` or `ChannelDimension`, *optional*):
+                The channel dimension format for the input image. Can be one of:
+                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                If unset, will use the inferred format of the input image.
+
+        Returns:
+            `np.ndarray`: The padded image.
+
+        """
+
+        # call the general `pad` if padding on `height/width`, otherwise it's the `num_patched` dim
+        if isinstance(padding, int) or len(padding) != 4:
+            return pad(image, padding, mode, constant_values, data_format, input_data_format)
+
+        if input_data_format is None:
+            input_data_format = infer_channel_dimension_format(image)
+        if mode == PaddingMode.CONSTANT:
+            image = np.pad(image, padding, mode="constant", constant_values=constant_values)
+        elif mode == PaddingMode.REFLECT:
+            image = np.pad(image, padding, mode="reflect")
+        elif mode == PaddingMode.REPLICATE:
+            image = np.pad(image, padding, mode="edge")
+        elif mode == PaddingMode.SYMMETRIC:
+            image = np.pad(image, padding, mode="symmetric")
+        else:
+            raise ValueError(f"Invalid padding mode: {mode}")
+        image = (
+            to_channel_dimension_format(image, data_format, input_data_format) if data_format is not None else image
+        )
+        return image
+
+    # Copied from transformers.models.llava_next.image_processing_llava_next.LlavaNextImageProcessor._resize_for_patching
+    def _resize_for_patching(
+        self, image: np.array, target_resolution: tuple, resample, input_data_format: ChannelDimension
+    ) -> np.array:
+        """
+        Resizes an image to a target resolution while maintaining aspect ratio.
+
+        Args:
+            image (np.array):
+                The input image.
+            target_resolution (tuple):
+                The target resolution (height, width) of the image.
+            resample (`PILImageResampling`):
+                Resampling filter to use if resizing the image.
+            input_data_format (`ChannelDimension` or `str`):
+                The channel dimension format of the input image.
+
+        Returns:
+            np.array: The resized and padded image.
+        """
+        
+        new_height, new_width = _get_patch_output_size(image, target_resolution, input_data_format)
+        # Resize the image
+        resized_image = resize(image, (new_height, new_width), resample=resample, input_data_format=input_data_format)
+
+        return resized_image
+
+    # Copied from transformers.models.llava_next.image_processing_llava_next.LlavaNextImageProcessor._pad_for_patching
+    def _pad_for_patching(
+        self, image: np.array, target_resolution: tuple, input_data_format: ChannelDimension
+    ) -> np.array:
+        """
+        Pad an image to a target resolution while maintaining aspect ratio.
+        """
+        target_height, target_width = target_resolution
+        new_height, new_width = _get_patch_output_size(image, target_resolution, input_data_format)
+
+        paste_x = (target_width - new_width) // 2
+        paste_y = (target_height - new_height) // 2
+
+        padded_image = self.pad(image, padding=((paste_y, paste_y), (paste_x, paste_x)))
+
+        return padded_image
+
+
+    def find_closest_aspect_ratio(self, aspect_ratio, target_ratios, width, height, image_size):
+        """
+        previous version mainly foucs on ratio.
+        We also consider area ratio here.
+        """
+        best_factor = float('-inf')
+        best_ratio = (1, 1)
+        area = width * height
+        for ratio in target_ratios:
+            target_aspect_ratio = ratio[0] / ratio[1]
+            ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+            area_ratio = (ratio[0]*ratio[1]*image_size*image_size)/ area
+            """
+            new area > 60% of original image area is enough.
+            """
+            factor_based_on_area_n_ratio = min((ratio[0]*ratio[1]*image_size*image_size)/ area, 0.6)* \
+                                     min(target_aspect_ratio/aspect_ratio, aspect_ratio/target_aspect_ratio)
+        
+            if factor_based_on_area_n_ratio > best_factor:
+                best_factor = factor_based_on_area_n_ratio
+                best_ratio = ratio
+        
+        return best_ratio
+
+
+    def get_image_patches(
+        self,
+        image: np.array,
+        min_num: int,
+        max_num: int,
+        size: tuple,
+        tile_size: int,
+        use_thumbnail: bool,
+        resample: PILImageResampling,
+        data_format: ChannelDimension,
+        input_data_format: ChannelDimension,
+    ):
+        image_size = get_image_size(image, channel_dim=input_data_format)
+        orig_height, orig_width = image_size
+        aspect_ratio = orig_width / orig_height
+
+        # calculate the existing image aspect ratio
+        target_ratios = set(
+            (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
+            i * j <= max_num and i * j >= min_num)
+        target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+        # find the closest aspect ratio to the target
+        target_aspect_ratio = self.find_closest_aspect_ratio(
+            aspect_ratio, target_ratios, orig_width, orig_height, tile_size)
+
+        # calculate the target width and height
+        target_width = tile_size * target_aspect_ratio[0]
+        target_height = tile_size * target_aspect_ratio[1]
+        blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+        if self.pad_during_tiling:
+            resized_image = self._resize_for_patching(
+                image, (target_height, target_width), resample=resample, input_data_format=input_data_format
+            )
+            padded_image = self._pad_for_patching(resized_image, (target_height, target_width), input_data_format=input_data_format)
+            image_used_to_split = padded_image
+        else:
+            image_used_to_split = resize(image, (target_height, target_width), resample=resample, input_data_format=input_data_format)
+
+        processed_tiles = []
+        for i in range(blocks):
+            box = (
+                (i % (target_width // tile_size)) * tile_size,
+                (i // (target_width // tile_size)) * tile_size,
+                ((i % (target_width // tile_size)) + 1) * tile_size,
+                ((i // (target_width // tile_size)) + 1) * tile_size
+            )
+            # split the image
+            split_img = crop(image_used_to_split, box[0], box[1], box[2], box[3], input_data_format)
+            processed_tiles.append(split_img)
+        assert len(processed_tiles) == blocks
+       
+        if use_thumbnail and len(processed_tiles) != 1:
+            thumbnail_img = resize(image, (tile_size, tile_size), resample=resample, input_data_format=input_data_format)
+            processed_tiles.append(thumbnail_img)
+
+        # make sure that all patches are in the input data format
+        processed_tiles = [
+            to_channel_dimension_format(tile, channel_dim=data_format, input_channel_dim=input_data_format)
+            for tile in processed_tiles
+        ]
+        return processed_tiles
+
+
+    # Copied from transformers.models.llava_next.image_processing_llava_next.LlavaNextImageProcessor._pad_for_batching
+    def _pad_for_batching(
+        self,
+        pixel_values: List[np.ndarray],
+        data_format: Optional[Union[str, ChannelDimension]] = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ):
+        """
+        Pads images on the `num_of_patches` dimension with zeros to form a batch of same number of patches.
+
+        Args:
+            pixel_values (`List[np.ndarray]`):
+                An array of pixel values of each images of shape (`batch_size`, `num_patches`, `image_in_3D`)
+            data_format (`str` or `ChannelDimension`, *optional*):
+                The channel dimension format for the output image. Can be one of:
+                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                If unset, will use same as the input image.
+            input_data_format (`str` or `ChannelDimension`, *optional*):
+                The channel dimension format for the input image. Can be one of:
+                    - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                    - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                If unset, will use the inferred format of the input image.
+
+        Returns:
+            List[`np.ndarray`]: The padded images.
+        """
+        max_patch = max(len(x) for x in pixel_values)
+        pixel_values = [
+            self.pad(
+                image,
+                padding=((0, max_patch - image.shape[0]), (0, 0), (0, 0), (0, 0)),
+                data_format=data_format,
+                input_data_format=input_data_format,
+            )
+            for image in pixel_values
+        ]
+
+        return pixel_values
+
+    def _preprocess(
+        self,
+        images: ImageInput,
+        do_resize: Optional[bool] = None,
+        size: Dict[str, int] = None,
+        resample: PILImageResampling = None,
+        do_rescale: Optional[bool] = None,
+        rescale_factor: Optional[float] = None,
+        do_normalize: Optional[bool] = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: Optional[bool] = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ) -> Image.Image:
+        """
+        Args:
+            images (`ImageInput`):
+                Batch of frames (one video) to preprocess. Expects a batch of frames with pixel values ranging from 0 to 255. If
+                passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+                Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
+                the longest edge resized to keep the input aspect ratio.
+            resample (`int`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+                has an effect if `do_resize` is set to `True`.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+                `True`.
+            data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. If unset, the channel dimension format is inferred
+                from the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+        """
+        if do_resize:
+            assert False, 'do_resize is not supported'
+            images = [
+                resize(image=image, size=size, resample=resample, input_data_format=input_data_format)
+                for image in images
+            ]
+
+        if do_rescale:
+            images = [
+                self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format)
+                for image in images
+            ]
+
+        if do_normalize:
+            images = [
+                self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format)
+                for image in images
+            ]
+
+        images = [
+            to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images
+        ]
+
+        return images
+
+    def preprocess(
+        self,
+        images: ImageInput,
+        do_resize: Optional[bool] = None,
+        size: Dict[str, int] = None,
+        resample: PILImageResampling = None,
+        do_rescale: Optional[bool] = None,
+        rescale_factor: Optional[float] = None,
+        do_normalize: Optional[bool] = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_pad: Optional[bool] = None,
+        do_convert_rgb: Optional[bool] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ):
+        """
+        Args:
+            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
+                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
+                tensor. Both channels-first and channels-last formats are supported.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+                Size of the image after resizing. Shortest edge of the image is resized to size["shortest_edge"], with
+                the longest edge resized to keep the input aspect ratio.
+            resample (`int`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+                has an effect if `do_resize` is set to `True`.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+                `True`.
+            do_pad (`bool`, *optional*, defaults to `self.do_pad`):
+                Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest
+                number of patches in the batch. Padding will be applied to the bottom and right with zeros.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            return_tensors (`str` or `TensorType`, *optional*):
+                The type of tensors to return. Can be one of:
+                - Unset: Return a list of `np.ndarray`.
+                - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
+                - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
+                - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+                - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
+            data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. If unset, the channel dimension format is inferred
+                from the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+
+        """
+        do_resize = do_resize if do_resize is not None else self.do_resize
+        size = size if size is not None else self.size
+        size = get_size_dict(size, default_to_square=False)
+        resample = resample if resample is not None else self.resample
+        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+        rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
+        do_normalize = do_normalize if do_normalize is not None else self.do_normalize
+        image_mean = image_mean if image_mean is not None else self.image_mean
+        image_std = image_std if image_std is not None else self.image_std
+        do_pad = do_pad if do_pad is not None else self.do_pad
+        do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+
+        images = make_flat_list_of_images(images)
+
+        if not valid_images(images):
+            raise ValueError(
+                "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
+                "torch.Tensor, tf.Tensor or jax.ndarray."
+            )
+
+        validate_preprocess_arguments(
+            do_rescale=do_rescale,
+            rescale_factor=rescale_factor,
+            do_normalize=do_normalize,
+            image_mean=image_mean,
+            image_std=image_std,
+            do_resize=do_resize,
+            size=size,
+            resample=resample,
+        )
+
+        if do_convert_rgb:
+            images = [convert_to_rgb(image) for image in images]
+
+        # All transformations expect numpy arrays.
+        images = [to_numpy_array(image) for image in images]
+
+        if do_rescale and is_scaled_image(images[0]):
+            logger.warning_once(
+                "It looks like you are trying to rescale already rescaled images. If the input"
+                " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
+            )
+
+        if input_data_format is None:
+            # We assume that all images have the same channel dimension format.
+            input_data_format = infer_channel_dimension_format(images[0])
+
+        processed_images = []
+        image_sizes = [get_image_size(image, channel_dim=input_data_format) for image in images]
+        for image in images:
+            # convert image into a list of patches
+            # we intentially use the same data format as the input data format
+            size_tuple = (
+                (size["height"], size["width"])
+                if "height" in size and "width" in size
+                else (size["shortest_edge"], size["shortest_edge"])
+            )
+            image_patches = self.get_image_patches(
+                image,
+                min_num=self.min_dynamic_tiles,
+                max_num=self.max_dynamic_tiles,
+                size=size_tuple,
+                tile_size=size["height"],
+                resample=resample,
+                data_format=input_data_format,
+                input_data_format=input_data_format,
+                use_thumbnail=self.use_thumbnail,
+            )
+
+            # preprocess patches
+            pixel_values = self._preprocess(
+                image_patches,
+                do_resize=do_resize,
+                size=size_tuple,
+                resample=resample,
+                do_rescale=do_rescale,
+                rescale_factor=rescale_factor,
+                do_normalize=do_normalize,
+                image_mean=image_mean,
+                image_std=image_std,
+                data_format=data_format,
+                input_data_format=input_data_format,
+            )
+            pixel_values = np.array(pixel_values)
+            processed_images.append(pixel_values)
+
+        if do_pad:
+            processed_images = self._pad_for_batching(processed_images)
+
+        return BatchFeature(
+            data={"pixel_values": processed_images, "image_sizes": image_sizes}, tensor_type=return_tensors
+        )
+
+
+__all__ = ["Eagle2ImageProcessor"]

image_processing_eagle2_5_vl_fast.py

@@ -0,0 +1,458 @@
+# --------------------------------------------------------
+# NVIDIA
+# Copyright (c) 2025 NVIDIA
+# Licensed under The MIT License [see LICENSE for details]
+# --------------------------------------------------------
+
+# copy from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_onevision/image_processing_llava_onevision_fast.py
+from typing import List, Optional, Union
+
+from transformers.image_processing_utils import BatchFeature, get_patch_output_size, select_best_resolution
+from transformers.image_processing_utils_fast import (
+    BASE_IMAGE_PROCESSOR_FAST_DOCSTRING,
+    BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS,
+    BaseImageProcessorFast,
+    DefaultFastImageProcessorKwargs,
+    divide_to_patches,
+    group_images_by_shape,
+    reorder_images,
+)
+from transformers.image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    IMAGENET_STANDARD_MEAN, # 0.5, 0.5, 0.5
+    IMAGENET_STANDARD_STD, # 0.5, 0.5, 0.5
+    ChannelDimension,
+    ImageInput,
+    VideoInput,
+    PILImageResampling,
+    SizeDict,
+    get_image_size,
+    make_flat_list_of_images,
+    make_batched_videos,
+    validate_kwargs
+)
+from transformers.processing_utils import Unpack
+from transformers.utils import TensorType, add_start_docstrings, is_torch_available, is_torchvision_v2_available
+
+
+if is_torch_available():
+    import torch
+if is_torchvision_v2_available():
+    from transformers.image_utils import pil_torch_interpolation_mapping
+
+    from torchvision.transforms.v2 import functional as F
+else:
+    from torchvision.transforms import functional as F
+
+def crop(img: torch.Tensor, left: int, top: int, right: int, bottom: int) -> torch.Tensor:
+    """Crop the given numpy array.
+    
+    Args:
+        img (torch.Tensor): Image to be cropped. Format should be (C, H, W).
+        left (int): The left coordinate of the crop box.
+        top (int): The top coordinate of the crop box.
+        right (int): The right coordinate of the crop box.
+        bottom (int): The bottom coordinate of the crop box.
+        
+    Returns:
+        torch.Tensor: Cropped image.
+    """
+    if not isinstance(img, torch.Tensor):
+        raise TypeError('img should be torch.Tensor. Got {}'.format(type(img)))
+    
+    if img.ndim not in [2, 3]:
+        raise ValueError('Image should have 2 or 3 dimensions. Got {}'.format(img.ndim))
+    
+    img_height = img.shape[1]
+    img_width = img.shape[2]
+    if top < 0 or left < 0 or bottom > img_height or right > img_width:
+        raise ValueError('Crop coordinates out of bounds')
+    
+    if top >= bottom or left >= right:
+        raise ValueError('Invalid crop coordinates')
+
+    return img[:, top:bottom, left:right]
+
+
+class Eagle2_5_VLFastImageProcessorKwargs(DefaultFastImageProcessorKwargs):
+    max_dynamic_tiles: Optional[int]
+    min_dynamic_tiles: Optional[int]
+    use_thumbnail: Optional[bool]
+    pad_during_tiling: Optional[bool]
+    do_pad: Optional[bool]
+
+
+@add_start_docstrings(
+    "Constructs a fast ConvNeXT image processor. Based on [`SiglipImageProcessor`] with incorporation of processing each video frame.",
+    BASE_IMAGE_PROCESSOR_FAST_DOCSTRING,
+    """
+        image_grid_pinpoints (`List[List[int]]`, *optional*):
+            A list of possible resolutions to use for processing high resolution images. The best resolution is selected
+            based on the original size of the image. Can be overridden by `image_grid_pinpoints` in the `preprocess`
+            method. Not used for processing videos.
+        do_pad (`bool`, *optional*):
+            Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest
+            number of patches in the batch. Padding will be applied to the bottom and right with zeros.
+    """,
+)
+class Eagle2_5_VLImageProcessorFast(BaseImageProcessorFast):
+    resample = PILImageResampling.BICUBIC
+    image_mean = IMAGENET_STANDARD_MEAN
+    image_std = IMAGENET_STANDARD_STD
+    size = {"height": 448, "width": 448}
+    default_to_square = False
+    crop_size = None
+    do_resize = True
+    do_center_crop = None
+    do_rescale = True
+    do_normalize = True
+    do_convert_rgb = True
+    do_pad = True
+    max_dynamic_tiles = 12
+    min_dynamic_tiles = 1
+    use_thumbnail = True
+    pad_during_tiling = False
+    valid_kwargs = Eagle2_5_VLFastImageProcessorKwargs
+    model_input_names = ["pixel_values_videos"]
+
+    def __init__(self, **kwargs: Unpack[Eagle2_5_VLFastImageProcessorKwargs]):
+        super().__init__(**kwargs)
+
+    @add_start_docstrings(
+        BASE_IMAGE_PROCESSOR_FAST_DOCSTRING_PREPROCESS,
+        """
+            max_dynamic_tiles (`int`, *optional*):
+                The maximum number of dynamic tiles to use for processing high resolution images.
+            min_dynamic_tiles (`int`, *optional*):
+                The minimum number of dynamic tiles to use for processing high resolution images.
+            use_thumbnail (`bool`, *optional*):
+                Whether to use a thumbnail for processing high resolution images.
+            pad_during_tiling (`bool`, *optional*):
+                Whether to pad the image during tiling.
+            do_pad (`bool`, *optional*):
+                    Whether to pad the image. If `True`, will pad the patch dimension of the images in the batch to the largest
+                    number of patches in the batch. Padding will be applied to the bottom and right with zeros.
+        """,
+    )
+    def preprocess(self, images: ImageInput, **kwargs: Unpack[Eagle2_5_VLFastImageProcessorKwargs]) -> BatchFeature:
+        return super().preprocess(images, **kwargs)
+
+    def _prepare_images_structure(
+        self,
+        images: ImageInput,
+    ) -> ImageInput:
+        """
+        Prepare the images structure for processing.
+
+        Args:
+            images (`ImageInput`):
+                The input images to process.
+
+        Returns:
+            `ImageInput`: The images with a valid nesting.
+        """
+        return make_flat_list_of_images(images)
+
+    def _prepare_videos_structure(self, videos: VideoInput) -> VideoInput:
+        return self._prepare_images_structure(videos)
+
+    def _prepare_input_videos(
+        self,
+        videos: VideoInput,
+        do_convert_rgb: Optional[bool] = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+        device: Optional["torch.device"] = None,
+    ) -> list["torch.Tensor"]:
+        """
+        Prepare the input images for processing.
+        """
+        videos = self._prepare_videos_structure(videos)
+        process_video_fn = partial(
+            self._process_image,
+            do_convert_rgb=do_convert_rgb,
+            input_data_format=input_data_format,
+            device=device,
+        )
+        # todo: yoni - check if we can parallelize this efficiently
+        processed_videos = []
+        for video in videos:
+            processed_videos.append(process_video_fn(video))
+
+        return processed_videos
+
+    def _resize_for_patching(
+        self,
+        image: "torch.Tensor",
+        target_resolution: tuple,
+        interpolation: "F.InterpolationMode",
+        input_data_format: ChannelDimension,
+    ) -> "torch.Tensor":
+        """
+        Resizes an image to a target resolution while maintaining aspect ratio.
+
+        Args:
+            image ("torch.Tensor"):
+                The input image.
+            target_resolution (tuple):
+                The target resolution (height, width) of the image.
+            interpolation (`InterpolationMode`):
+                Resampling filter to use if resizing the image.
+            input_data_format (`ChannelDimension` or `str`):
+                The channel dimension format of the input image.
+
+        Returns:
+            "torch.Tensor": The resized and padded image.
+        """
+        new_height, new_width = get_patch_output_size(image, target_resolution, input_data_format)
+
+        # Resize the image
+        resized_image = F.resize(image, (new_height, new_width), interpolation=interpolation)
+
+        return resized_image
+
+    def find_closest_aspect_ratio(self, aspect_ratio, target_ratios, width, height, image_size):
+        """
+        previous version mainly foucs on ratio.
+        We also consider area ratio here.
+        """
+        best_factor = float('-inf')
+        best_ratio = (1, 1)
+        area = width * height
+        for ratio in target_ratios:
+            target_aspect_ratio = ratio[0] / ratio[1]
+            ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+            area_ratio = (ratio[0]*ratio[1]*image_size*image_size)/ area
+            """
+            new area > 60% of original image area is enough.
+            """
+            factor_based_on_area_n_ratio = min((ratio[0]*ratio[1]*image_size*image_size)/ area, 0.6)* \
+                                     min(target_aspect_ratio/aspect_ratio, aspect_ratio/target_aspect_ratio)
+        
+            if factor_based_on_area_n_ratio > best_factor:
+                best_factor = factor_based_on_area_n_ratio
+                best_ratio = ratio
+        
+        return best_ratio
+    
+    def _pad_for_patching(
+        self, image: "torch.Tensor", target_resolution: tuple, input_data_format: ChannelDimension
+    ) -> "torch.Tensor":
+        """
+        Pad an image to a target resolution while maintaining aspect ratio.
+        """
+        target_height, target_width = target_resolution
+        new_height, new_width = get_patch_output_size(image, target_resolution, input_data_format)
+
+        paste_x = (target_width - new_width) // 2
+        paste_y = (target_height - new_height) // 2
+
+        padded_image = F.pad(image, padding=[paste_x, paste_y, paste_x, paste_y])
+
+        return padded_image
+
+    def _get_image_patches(
+        self,
+        image: "torch.Tensor",
+        min_num: int,
+        max_num: int,
+        size: tuple,
+        tile_size: int,
+        use_thumbnail: bool,
+        interpolation: "F.InterpolationMode",
+        pad_during_tiling: bool,
+    ) -> List["torch.Tensor"] :
+        image_size = get_image_size(image, channel_dim=ChannelDimension.FIRST)
+        orig_height, orig_width = image_size
+        aspect_ratio = orig_width / orig_height
+
+        # calculate the existing image aspect ratio
+        target_ratios = set(
+            (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
+            i * j <= max_num and i * j >= min_num)
+        target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+        # find the closest aspect ratio to the target
+        target_aspect_ratio = self.find_closest_aspect_ratio(
+            aspect_ratio, target_ratios, orig_width, orig_height, tile_size)
+
+        # calculate the target width and height
+        target_width = tile_size * target_aspect_ratio[0]
+        target_height = tile_size * target_aspect_ratio[1]
+        blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+        if pad_during_tiling:
+            resized_image = self._resize_for_patching(
+                image, (target_height, target_width), interpolation=interpolation, input_data_format=ChannelDimension.FIRST
+            )
+            padded_image = self._pad_for_patching(resized_image, (target_height, target_width), input_data_format=ChannelDimension.FIRST)
+            image_used_to_split = padded_image
+        else:
+            image_used_to_split = F.resize(image, (target_height, target_width), interpolation=interpolation)
+
+        processed_tiles = []
+        for i in range(blocks):
+            box = (
+                (i % (target_width // tile_size)) * tile_size,
+                (i // (target_width // tile_size)) * tile_size,
+                ((i % (target_width // tile_size)) + 1) * tile_size,
+                ((i // (target_width // tile_size)) + 1) * tile_size
+            )
+            # split the image
+            split_img = crop(image_used_to_split, box[0], box[1], box[2], box[3])
+            processed_tiles.append(split_img)
+        assert len(processed_tiles) == blocks
+       
+        if use_thumbnail and len(processed_tiles) != 1:
+            thumbnail_img = F.resize(image, (tile_size, tile_size), interpolation=interpolation)
+            processed_tiles.append(thumbnail_img)
+
+        return processed_tiles
+
+    def _pad_for_batching(
+        self,
+        pixel_values: List["torch.Tensor"],
+    ) -> List["torch.Tensor"]:
+        """
+        Pads images on the `num_of_patches` dimension with zeros to form a batch of same number of patches.
+
+        Args:
+            pixel_values (`List[torch.Tensor]`):
+                An array of pixel values of each images of shape (`batch_size`, `num_patches`, `image_in_3D`)
+
+        Returns:
+            List[`torch.Tensor`]: The padded images.
+        """
+        max_patch = max(len(x) for x in pixel_values)
+        pixel_values = [
+            torch.nn.functional.pad(image, pad=[0, 0, 0, 0, 0, 0, 0, max_patch - image.shape[0]])
+            for image in pixel_values
+        ]
+
+        return pixel_values
+
+    def _preprocess(
+        self,
+        images: List["torch.Tensor"],
+        do_resize: bool,
+        size: SizeDict,
+        max_dynamic_tiles: int,
+        min_dynamic_tiles: int,
+        use_thumbnail: bool,
+        pad_during_tiling: bool,
+        interpolation: Optional["F.InterpolationMode"],
+        do_center_crop: bool,
+        crop_size: SizeDict,
+        do_rescale: bool,
+        rescale_factor: float,
+        do_normalize: bool,
+        image_mean: Optional[Union[float, List[float]]],
+        image_std: Optional[Union[float, List[float]]],
+        do_pad: bool,
+        return_tensors: Optional[Union[str, TensorType]],
+    ) -> BatchFeature:
+        processed_images = []
+        image_sizes = []
+        # Determine the size tuple
+        if size and size.height and size.width:
+            size_tuple = (size.height, size.width)
+        else:
+            size_tuple = (size.shortest_edge, size.shortest_edge)
+
+        # Determine the patch size
+        if crop_size and crop_size.height:
+            tile_size = crop_size.height
+        elif size and size.height:
+            tile_size = size.height
+        else:
+            tile_size = size.shortest_edge
+
+        for image in images:
+            image_patches = self._get_image_patches(
+                image,
+                min_num=min_dynamic_tiles,
+                max_num=max_dynamic_tiles,
+                size=size_tuple,
+                tile_size=tile_size,
+                use_thumbnail=use_thumbnail,
+                interpolation=interpolation,
+                pad_during_tiling=pad_during_tiling,
+            )
+
+            # Group images by size for batched processing
+            processed_image_patches_grouped = {}
+            grouped_image_patches, grouped_image_patches_index = group_images_by_shape(image_patches)
+
+            for shape, stacked_image_patches in grouped_image_patches.items():
+                if do_resize:
+                    stacked_image_patches = self.resize(
+                        image=stacked_image_patches,
+                        size=size,
+                        interpolation=interpolation,
+                    )
+                if do_center_crop:
+                    stacked_image_patches = self.center_crop(stacked_image_patches, crop_size)
+                # Fused rescale and normalize
+                stacked_image_patches = self.rescale_and_normalize(
+                    stacked_image_patches, do_rescale, rescale_factor, do_normalize, image_mean, image_std
+                )
+                processed_image_patches_grouped[shape] = stacked_image_patches
+            processed_image_patches = reorder_images(processed_image_patches_grouped, grouped_image_patches_index)
+            processed_image_patches = (
+                torch.stack(processed_image_patches, dim=0) if return_tensors else processed_image_patches
+            )
+            processed_images.append(processed_image_patches)
+            image_sizes.append(get_image_size(image, ChannelDimension.FIRST))
+
+        if do_pad:
+            processed_images = self._pad_for_batching(processed_images)
+        
+        # processed_images = torch.stack(processed_images, dim=0) if return_tensors else processed_images
+        processed_images = torch.cat(processed_images, dim=0) if return_tensors else processed_images
+        return BatchFeature(
+            data={"pixel_values": processed_images, "image_sizes": image_sizes}, tensor_type=return_tensors
+        )
+
+
+    def preprocess(self, images: ImageInput, videos: VideoInput=None, **kwargs: Unpack[Eagle2_5_VLFastImageProcessorKwargs]) -> BatchFeature:
+        validate_kwargs(captured_kwargs=kwargs.keys(), valid_processor_keys=self.valid_kwargs.__annotations__.keys())
+        # Set default kwargs from self. This ensures that if a kwarg is not provided
+        # by the user, it gets its default value from the instance, or is set to None.
+        for kwarg_name in self.valid_kwargs.__annotations__:
+            kwargs.setdefault(kwarg_name, getattr(self, kwarg_name, None))
+
+        # Extract parameters that are only used for preparing the input images
+        do_convert_rgb = kwargs.pop("do_convert_rgb")
+        input_data_format = kwargs.pop("input_data_format")
+        device = kwargs.pop("device")
+        # Prepare input images
+        if images is not None:
+            images = self._prepare_input_images(
+                images=images, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device
+            )
+
+        if videos is not None:
+            videos = self._prepare_input_images(
+                images=videos, do_convert_rgb=do_convert_rgb, input_data_format=input_data_format, device=device
+            )
+
+        # Update kwargs that need further processing before being validated
+        kwargs = self._further_process_kwargs(**kwargs)
+
+        # Validate kwargs
+        self._validate_preprocess_kwargs(**kwargs)
+
+        # torch resize uses interpolation instead of resample
+        resample = kwargs.pop("resample")
+        kwargs["interpolation"] = (
+            pil_torch_interpolation_mapping[resample] if isinstance(resample, (PILImageResampling, int)) else resample
+        )
+
+        # Pop kwargs that are not needed in _preprocess
+        kwargs.pop("default_to_square")
+        kwargs.pop("data_format")
+        if images is not None:
+            return self._preprocess(images, **kwargs)
+        elif videos is not None:
+            return self._preprocess(videos, **kwargs)
+    
+__all__ = ["Eagle2_5_VLImageProcessorFast"]

modeling_eagle_chat.py → modeling_eagle2_5_vl.py

@@ -1,84 +1,118 @@
 # --------------------------------------------------------
-# Eagle2
+# NVIDIA
 # Copyright (c) 2025 NVIDIA
-# Licensed under The Apache License [see LICENSE for details]
+# Licensed under The MIT License [see LICENSE for details]
 # --------------------------------------------------------
 
 import warnings
+import inspect
 from typing import Any, List, Optional, Tuple, Union
-
-import torch.utils.checkpoint
-import transformers
+import torch
 from torch import nn
+import torch.distributed as dist
 from torch.nn import CrossEntropyLoss
-from transformers import (AutoModel, GenerationConfig,
-                          LlamaTokenizer, LlamaForCausalLM)
+import torch.nn.functional as F
+from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM
+from transformers.models.llama.modeling_llama import LlamaForCausalLM
+import torch.utils.checkpoint as cp
+from transformers.models.siglip.modeling_siglip import SiglipVisionModel
+from peft import LoraConfig, get_peft_model
+from transformers.generation import GenerationMixin
+from transformers import GenerationConfig
 from transformers.modeling_outputs import CausalLMOutputWithPast
 from transformers.modeling_utils import PreTrainedModel
 from transformers.utils import ModelOutput, logging
-from peft import LoraConfig, get_peft_model
-from transformers.models.siglip.modeling_siglip import SiglipVisionModel
-
-from transformers.models.qwen2.modeling_qwen2 import Qwen2ForCausalLM
+from .configuration_eagle2_5_vl import Eagle2_5_VLConfig
+from transformers.utils import add_start_docstrings, add_start_docstrings_to_model_forward, logging, replace_return_docstrings
 
 logger = logging.get_logger(__name__)
-from .configuration_eagle_chat import Eagle2ChatConfig
-
-def version_cmp(v1, v2, op='eq'):
-    import operator
 
-    from packaging import version
-    op_func = getattr(operator, op)
-    return op_func(version.parse(v1), version.parse(v2))
 
-
-class Eagle2ChatModel(PreTrainedModel):
-    config_class = Eagle2ChatConfig
-    main_input_name = 'pixel_values'
-    _no_split_modules = ['LlamaDecoderLayer']
+# copy from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_onevision/modeling_llava_onevision.py#L241C1-L280C1
+EAGLE2_5_VL_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+
+    Parameters:
+        config ([`Eagle2_5_VLConfig`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+@add_start_docstrings(
+    "The bare Eagle2_5_VL Model outputting raw hidden-states without any specific head on top.",
+    EAGLE2_5_VL_START_DOCSTRING,
+)
+class Eagle2_5_VLPreTrainedModel(PreTrainedModel):
+    config_class = Eagle2_5_VLConfig
+    base_model_prefix = "model"
+    main_input_name = 'input_ids'
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Qwen2DecoderLayer", "LlamaDecoderLayer" ,"Siglip2EncoderLayer", "SiglipEncoderLayer"]
+    _skip_keys_device_placement = "past_key_values"
     _supports_flash_attn_2 = True
+    _supports_cache_class = True
+    _supports_static_cache = True
+    _supports_quantized_cache = True
     _supports_sdpa = True
-    _supports_flex_attn = False
-    _supports_cache_class = False
-    _supports_quantized_cache = False
-    _supports_static_cache = False
-    _supports_attention_backend = False
     
-    def __init__(self, config: Eagle2ChatConfig, vision_model=None, language_model=None):
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, (nn.Linear, nn.Conv2d)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+class Eagle2_5_VLForConditionalGeneration(Eagle2_5_VLPreTrainedModel, GenerationMixin):
+    config_class = Eagle2_5_VLConfig
+    def __init__(self, config: Eagle2_5_VLConfig, vision_model=None, language_model=None):
         super().__init__(config)
 
         image_size = config.force_image_size or config.vision_config.image_size
-
         patch_size = config.vision_config.patch_size
         self.patch_size = patch_size
         self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
 
         self.select_layer = config.select_layer
-        self.template = config.template
         self.downsample_ratio = config.downsample_ratio
-
+        self.loss_version = config.loss_version
+        self.mlp_checkpoint = config.mlp_checkpoint
+        
         logger.info(f'num_image_token: {self.num_image_token}')
+        logger.info(f'mlp_checkpoint: {self.mlp_checkpoint}')
         if vision_model is not None:
             self.vision_model = vision_model
         else:
             if config.vision_config.model_type == 'siglip_vision_model':
-                if version_cmp(transformers.__version__, '4.43.0', 'le'):
-                    config.vision_config._attn_implementation = 'eager'
+                config.vision_config._attn_implementation = 'flash_attention_2'
                 self.vision_model = SiglipVisionModel(config.vision_config)
+            else:
+                raise NotImplementedError(f'{config.vision_config.model_type} is not implemented.')
 
         if language_model is not None:
             self.language_model = language_model
         else:
-            if config.llm_config.architectures[0] == 'LlamaForCausalLM':
-                self.language_model = LlamaForCausalLM(config.llm_config)
-            elif config.llm_config.architectures[0] == 'Qwen2ForCausalLM':
-                self.language_model = Qwen2ForCausalLM(config.llm_config)
+            if config.text_config.architectures[0] == 'LlamaForCausalLM':
+                self.language_model = LlamaForCausalLM(config.text_config)
+            elif config.text_config.architectures[0] == 'Qwen2ForCausalLM':
+                # assert config.text_config._attn_implementation == 'flash_attention_2', f"Qwen2 must use flash_attention_2 but got {config.text_config._attn_implementation}"
+                self.language_model = Qwen2ForCausalLM(config.text_config)
             else:
-                raise NotImplementedError(f'{config.llm_config.architectures[0]} is not implemented.')
+                raise NotImplementedError(f'{config.text_config.architectures[0]} is not implemented.')
 
         vit_hidden_size = config.vision_config.hidden_size
-
-        llm_hidden_size = config.llm_config.hidden_size
+        llm_hidden_size = config.text_config.hidden_size
 
         self.mlp1 = nn.Sequential(
                 nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio) ** 2),
@@ -86,26 +120,39 @@ class Eagle2ChatModel(PreTrainedModel):
                 nn.GELU(),
                 nn.Linear(llm_hidden_size, llm_hidden_size)
             )
-        self.img_context_token_id = None
-        self.system_message = 'You are a helpful assistant.' # Default system message
+        self.image_token_index = config.image_token_index
+        self.neftune_alpha = None
+
 
         if config.use_backbone_lora:
             self.wrap_backbone_lora(r=config.use_backbone_lora, lora_alpha=2 * config.use_backbone_lora)
 
+        self.use_llm_lora = config.use_llm_lora 
         if config.use_llm_lora:
             self.wrap_llm_lora(r=config.use_llm_lora, lora_alpha=2 * config.use_llm_lora)
-    
+            
+        self.check_forward_kwargs()
+        
+    def check_forward_kwargs(self):
+        # We intentionally avoid using **kwargs in forward because Hugging Face Transformers
+        # has special handling for functions with **kwargs parameters that would affect
+        # how our model is processed during training and inference.
+        forward_params = inspect.signature(self.forward).parameters
+        assert not any(k.kind == inspect.Parameter.VAR_KEYWORD for k in forward_params.values())
+
+        
     def wrap_backbone_lora(self, r=128, lora_alpha=256, lora_dropout=0.05):
         lora_config = LoraConfig(
             r=r,
-            target_modules=['attn.qkv', 'attn.proj', 'mlp.fc1', 'mlp.fc2'],
+            target_modules=['self_attn.q_proj', 'self_attn.k_proj', 'self_attn.v_proj', 'self_attn.out_proj',
+                            'mlp.fc1', 'mlp.fc2'],
             lora_alpha=lora_alpha,
             lora_dropout=lora_dropout,
         )
         self.vision_model = get_peft_model(self.vision_model, lora_config)
         self.vision_model.print_trainable_parameters()
 
-    def wrap_llm_lora(self, r=128, lora_alpha=256, lora_dropout=0.05):    
+    def wrap_llm_lora(self, r=128, lora_alpha=256, lora_dropout=0.05):
         lora_config = LoraConfig(
             r=r,
             target_modules=['self_attn.q_proj', 'self_attn.k_proj', 'self_attn.v_proj', 'self_attn.o_proj',
@@ -117,8 +164,8 @@ class Eagle2ChatModel(PreTrainedModel):
         self.language_model = get_peft_model(self.language_model, lora_config)
         self.language_model.enable_input_require_grads()
         self.language_model.print_trainable_parameters()
-
-
+        self.use_llm_lora = True
+        
     def forward(
             self,
             pixel_values: torch.FloatTensor,
@@ -132,7 +179,7 @@ class Eagle2ChatModel(PreTrainedModel):
             output_attentions: Optional[bool] = None,
             output_hidden_states: Optional[bool] = None,
             return_dict: Optional[bool] = None,
-            num_patches_list: Optional[List[torch.Tensor]] = None,
+            num_tiles_list: Optional[List[torch.Tensor]] = None,
     ) -> Union[Tuple, CausalLMOutputWithPast]:
         return_dict = return_dict if return_dict is not None else self.config.use_return_dict
 
@@ -154,7 +201,7 @@ class Eagle2ChatModel(PreTrainedModel):
             print(f'dynamic ViT batch size: {vit_batch_size}, images per sample: {vit_batch_size / B}, dynamic token length: {N}')
 
         input_ids = input_ids.reshape(B * N)
-        selected = (input_ids == self.img_context_token_id)
+        selected = (input_ids == self.image_token_index)
         try:
             input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds.reshape(-1, C)
         except Exception as e:
@@ -211,165 +258,36 @@ class Eagle2ChatModel(PreTrainedModel):
         # N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2)
         x = x.view(n, int(h * scale_factor), int(w * scale_factor),
                    int(c / (scale_factor * scale_factor)))
+
         x = x.permute(0, 2, 1, 3).contiguous()
         return x
 
     def extract_feature(self, pixel_values):
-        """
-        """
-        
         if self.select_layer == -1:
             vit_embeds = self.vision_model(
                 pixel_values=pixel_values,
                 output_hidden_states=False,
                 return_dict=True)
-            # if there is vit_embeds.last_hidden_state, use it.
             if hasattr(vit_embeds, 'last_hidden_state'):
                 vit_embeds = vit_embeds.last_hidden_state
+            
         else:
             vit_embeds = self.vision_model(
                 pixel_values=pixel_values,
                 output_hidden_states=True,
                 return_dict=True).hidden_states[self.select_layer]
-        if type(self.vision_model) == SiglipVisionModel:
-            pass
-        else:
-            vit_embeds = vit_embeds[:, 1:, :] # torch.Size([B, 1024, 1024])
-
-        if self.training and self.neftune_alpha is not None:
-            vit_embeds = self.noised_embed(vit_embeds, self.neftune_alpha)
-
-
+        
         h = w = int(vit_embeds.shape[1] ** 0.5)
         vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
-
         vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio) # torch.Size([B, 1024, 1024]) -> torch.Size([B, 16, 16, 4096])
         vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1]) # torch.Size([B, 16, 16, 4096]) -> torch.Size([B, 256, 4096])
-        vit_embeds = self.mlp1(vit_embeds)#.to(pixel_values.device)
+        if self.mlp_checkpoint and vit_embeds.requires_grad:
+            vit_embeds = cp.checkpoint(self.mlp1, vit_embeds)
+        else:
+            vit_embeds = self.mlp1(vit_embeds)
 
         return vit_embeds
-    
-    def batch_chat(self, 
-                   tokenizer, pixel_values, questions, generation_config,     num_patches_list=None,
-                   history=None, return_history=False, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>',
-                   IMG_CONTEXT_TOKEN='<IMG_CONTEXT>', verbose=False, image_counts=None):
-        if history is not None or return_history:
-            print('Now multi-turn chat is not supported in batch_chat.')
-            raise NotImplementedError
-
-        if image_counts is not None:
-            num_patches_list = image_counts
-            print('Warning: `image_counts` is deprecated. Please use `num_patches_list` instead.')
-
-        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
-        self.img_context_token_id = img_context_token_id
-
-        if verbose and pixel_values is not None:
-            image_bs = pixel_values.shape[0]
-            print(f'dynamic ViT batch size: {image_bs}')
-
-        queries = []
-        for idx, num_patches in enumerate(num_patches_list):
-            question = questions[idx]
-            if pixel_values is not None and '<image>' not in question:
-                question = '<image>\n' + question
-            template_messages = []
-            sep = tokenizer.eos_token
-            template_messages.append(('<|im_start|>user', question))
-            template_messages.append(('<|im_end|>assistant', None))
-            query = self.get_prompt(self.system_message, template_messages, sep)
-
-            image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + IMG_END_TOKEN
-            query = query.replace('<image>', image_tokens, 1)
-            queries.append(query)
-
-        tokenizer.padding_side = 'left'
-        model_inputs = tokenizer(queries, return_tensors='pt', padding=True)
-        input_ids = model_inputs['input_ids'].cuda()
-        attention_mask = model_inputs['attention_mask'].cuda()
-        eos_token_id = tokenizer.convert_tokens_to_ids(sep)
-        generation_config['eos_token_id'] = eos_token_id
-        generation_output = self.generate(
-            pixel_values=pixel_values,
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            **generation_config
-        )
-        responses = tokenizer.batch_decode(generation_output, skip_special_tokens=True)
-        responses = [response.split(sep)[0].strip() for response in responses]
-        return responses
-
-    def chat(self, tokenizer, pixel_values, question, generation_config, history=None, return_history=False,
-             num_patches_list=None, IMG_START_TOKEN='<img>', IMG_END_TOKEN='</img>', IMG_CONTEXT_TOKEN='<IMG_CONTEXT>',
-             verbose=False, llm_only=False):
-
-        if history is None and pixel_values is not None and '<image>' not in question:
-            question = '<image>\n' + question
-
-        if num_patches_list is None:
-            num_patches_list = [pixel_values.shape[0]] if pixel_values is not None else []
-        assert pixel_values is None or len(pixel_values) == sum(num_patches_list)
-
-        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
-        self.img_context_token_id = img_context_token_id
-
-        template_messages = []
-        system_message = f'<|im_start|>system\n{self.system_message}'
-        sep = tokenizer.eos_token
-        eos_token_id = tokenizer.convert_tokens_to_ids(sep)
-
-        history = [] if history is None else history
-        for (old_question, old_answer) in history:
-            template_messages.append(('<|im_start|>user', old_question))
-            template_messages.append(('<|im_start|>assistant', old_answer))
-        template_messages.append(('<|im_start|>user', question))
-        template_messages.append(('<|im_end|>assistant', None))
-        query = self.get_prompt(system_message, template_messages, sep)
-
-        if verbose and pixel_values is not None:
-            image_bs = pixel_values.shape[0]
-            print(f'dynamic ViT batch size: {image_bs}')
-
-        for num_patches in num_patches_list:
-            image_tokens = IMG_START_TOKEN + IMG_CONTEXT_TOKEN * self.num_image_token * num_patches + IMG_END_TOKEN
-            if llm_only:
-                query = query.replace('<image>', '', 1)
-            else:
-                query = query.replace('<image>', image_tokens, 1)
-        
-        model_inputs = tokenizer(query, return_tensors='pt')
-        input_ids = model_inputs['input_ids'].cuda()
-        attention_mask = model_inputs['attention_mask'].cuda()
-        generation_config['eos_token_id'] = eos_token_id
-        generation_output = self.generate(
-            pixel_values=pixel_values,
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            **generation_config
-        )
-        response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
-        response = response.split(sep)[0].strip()
-        history.append((question, response))
-        if return_history:
-            return response, history
-        else:
-            query_to_print = query.replace(IMG_CONTEXT_TOKEN, '')
-            query_to_print = query_to_print.replace(f'{IMG_START_TOKEN}{IMG_END_TOKEN}', '<image>')
-            if verbose:
-                print(query_to_print, response)
-            return response
-
-    def get_prompt(self, system_prompt, messages, sep) -> str:
-        """Get the prompt for generation."""
-        
-        ret = '' if system_prompt == '' else system_prompt + sep + '\n'
-        for role, message in messages:
-            if message:
-                ret += role + '\n' + message + sep + '\n'
-            else:
-                ret += role + '\n'
-        return ret
-        
+
     @torch.no_grad()
     def generate(
             self,
@@ -379,11 +297,10 @@ class Eagle2ChatModel(PreTrainedModel):
             visual_features: Optional[torch.FloatTensor] = None,
             generation_config: Optional[GenerationConfig] = None,
             output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
+            image_sizes: Optional[List[Tuple[int, int]]] = None,
             **generate_kwargs,
     ) -> torch.LongTensor:
 
-        assert self.img_context_token_id is not None
         if pixel_values is not None:
             if visual_features is not None:
                 vit_embeds = visual_features
@@ -395,7 +312,7 @@ class Eagle2ChatModel(PreTrainedModel):
             input_embeds = input_embeds.reshape(B * N, C)
 
             input_ids = input_ids.reshape(B * N)
-            selected = (input_ids == self.img_context_token_id)
+            selected = (input_ids == self.config.image_token_index)
             assert selected.sum() != 0
             input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
 
@@ -413,9 +330,28 @@ class Eagle2ChatModel(PreTrainedModel):
         )
 
         return outputs
-        
+
+    # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.get_input_embeddings
     def get_input_embeddings(self):
         return self.language_model.get_input_embeddings()
-    
+
+    # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.set_input_embeddings
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.get_output_embeddings
     def get_output_embeddings(self):
-        return self.language_model.get_output_embeddings()
+        return self.language_model.get_output_embeddings()
+
+    # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.set_output_embeddings
+    def set_output_embeddings(self, new_embeddings):
+        self.language_model.set_output_embeddings(new_embeddings)
+
+    # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.set_decoder
+    def set_decoder(self, decoder):
+        self.language_model.set_decoder(decoder)
+
+    # Copied from transformers.models.llava_next.modeling_llava_next.LlavaNextForConditionalGeneration.get_decoder
+    def get_decoder(self):
+        return self.language_model.get_decoder()
+

preprocessor_config.json

@@ -0,0 +1,23 @@
+{
+    "auto_map": {
+        "AutoProcessor": "processing_eagle2_5_vl.Eagle2_5_VLProcessor",
+        "AutoImageProcessor": "image_processing_eagle2_5_vl_fast.Eagle2_5_VLImageProcessorFast"
+    },
+    "image_processor_type": "Eagle2_5_VLImageProcessorFast",
+    "processor_class": "Eagle2_5_VLProcessor",
+    "image_mean": [0.5, 0.5, 0.5],
+    "image_std": [0.5, 0.5, 0.5],
+    "do_resize": false,
+    "size": {
+        "height": 448,
+        "width": 448
+    },
+    "max_dynamic_tiles": 12,
+    "min_dynamic_tiles": 1,
+    "tokens_per_tile": 256,
+    "use_thumbnail": true,
+    "do_rescale": true,
+    "do_normalize": true,
+    "do_pad": false,
+    "do_convert_rgb": true
+}

processing_eagle2_5_vl.py

@@ -0,0 +1,738 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Processor class for Eagle2_5_VL.
+copy from https://github.com/huggingface/transformers/blob/main/src/transformers/models/llava_onevision/processing_llava_onevision.py
+"""
+
+import math
+import os
+from typing import Iterable, List, Union, Literal
+import base64
+import sys
+import time
+import warnings
+from functools import lru_cache
+from io import BytesIO
+import re
+import requests
+import torch
+import torchvision
+from packaging import version
+from PIL import Image
+from torchvision import io
+from typing import Optional, Any
+import numpy as np
+
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_processing_utils import select_best_resolution
+from transformers.image_utils import ImageInput, VideoInput, get_image_size, to_numpy_array
+from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
+from transformers.utils import logging
+from transformers.models.auto import AutoImageProcessor
+
+logger = logging.get_logger(__name__)
+
+
+
+FRAME_FACTOR = 2
+FPS = 2.0
+FPS_MIN_FRAMES = 4
+FPS_MAX_FRAMES = 256
+
+
+
+def adjust_by_factor(number: int, factor: int, method: Literal['round', 'ceil', 'floor'] = 'round') -> int:
+    """Adjusts 'number' to the nearest, ceiling, or floor multiple of 'factor'."""
+    op = {'round': round, 'ceil': math.ceil, 'floor': math.floor}[method]
+    return op(number / factor) * factor
+
+
+def to_rgb(pil_image: Image.Image) -> Image.Image:
+      if pil_image.mode == 'RGBA':
+          white_background = Image.new("RGB", pil_image.size, (255, 255, 255))
+          white_background.paste(pil_image, mask=pil_image.split()[3])  # Use alpha channel as mask
+          return white_background
+      else:
+          return pil_image.convert("RGB")
+
+
+def fetch_image(ele: dict[str, str | Image.Image]) -> Image.Image:
+    if "image" in ele:
+        image = ele["image"]
+    else:
+        image = ele["image_url"]
+    image_obj = None
+    if isinstance(image, Image.Image):
+        image_obj = image
+    elif image.startswith("http://") or image.startswith("https://"):
+        response = requests.get(image, stream=True)
+        image_obj = Image.open(BytesIO(response.content))
+    elif image.startswith("file://"):
+        image_obj = Image.open(image[7:])
+    elif image.startswith("data:image"):
+        if "base64," in image:
+            _, base64_data = image.split("base64,", 1)
+            data = base64.b64decode(base64_data)
+            image_obj = Image.open(BytesIO(data))
+    else:
+        image_obj = Image.open(image)
+    if image_obj is None:
+        raise ValueError(f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}")
+    image = to_rgb(image_obj)
+    if 'scale_factor' in ele:
+        scale_factor = ele['scale_factor']
+        image = image.resize((image.width * scale_factor, image.height * scale_factor), Image.BILINEAR)
+    return image
+
+
+def smart_nframes(
+    ele: dict,
+    total_frames: int,
+    video_fps: int | float,
+) -> int:
+    """calculate the number of frames for video used for model inputs.
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+            support either `fps` or `nframes`:
+                - nframes: the number of frames to extract for model inputs.
+                - fps: the fps to extract frames for model inputs.
+                    - min_frames: the minimum number of frames of the video, only used when fps is provided.
+                    - max_frames: the maximum number of frames of the video, only used when fps is provided.
+        total_frames (int): the original total number of frames of the video.
+        video_fps (int | float): the original fps of the video.
+
+    Raises:
+        ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
+
+    Returns:
+        int: the number of frames for video used for model inputs.
+    """
+    assert not ("fps" in ele and "nframes" in ele), "Only accept either `fps` or `nframes`"
+    if "nframes" in ele:
+        nframes = adjust_by_factor(ele["nframes"], FRAME_FACTOR, method='round')
+    else:
+        fps = ele.get("fps", FPS)
+        min_frames = adjust_by_factor(ele.get("min_frames", FPS_MIN_FRAMES), FRAME_FACTOR, method='ceil')
+        max_frames = adjust_by_factor(ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)), FRAME_FACTOR, method='floor')
+        nframes = total_frames / video_fps * fps
+        if nframes > total_frames:
+            logger.warning(f"smart_nframes: nframes[{nframes}] > total_frames[{total_frames}]")
+        nframes = min(min(max(nframes, min_frames), max_frames), total_frames)
+        nframes = adjust_by_factor(nframes, FRAME_FACTOR, method='floor')
+    if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
+        nframes = total_frames
+        # raise ValueError(f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}.")
+    return nframes
+
+
+def _read_video_torchvision(
+    ele: dict,
+) -> (torch.Tensor, float, list):
+    """read video using torchvision.io.read_video and return also per-frame timestamps"""
+    video_path = ele["video"]
+    if version.parse(torchvision.__version__) < version.parse("0.19.0"):
+        if "http://" in video_path or "https://" in video_path:
+            warnings.warn("torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0.")
+        if "file://" in video_path:
+            video_path = video_path[7:]
+    st = time.time()
+    video, audio, info = io.read_video(
+        video_path,
+        start_pts=ele.get("video_start", 0.0),
+        end_pts=ele.get("video_end", None),
+        pts_unit="sec",
+        output_format="TCHW",
+    )
+    total_frames, video_fps = video.size(0), info["video_fps"]
+    logger.info(f"torchvision:  {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
+    nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
+    # Calculate frame indices and corresponding timestamps (based on video start time)
+    idx = torch.linspace(0, total_frames - 1, nframes).round().long()
+    start_time = ele.get("video_start", 0.0)
+    timestamps = (start_time + idx.to(torch.float32) / video_fps).tolist()
+    sample_fps = nframes / max(total_frames, 1e-6) * video_fps
+    video = video[idx]
+    return video, sample_fps, timestamps
+
+
+
+def is_decord_available() -> bool:
+    import importlib.util
+
+    return importlib.util.find_spec("decord") is not None
+
+def _read_video_decord(
+    ele: dict,
+) -> (torch.Tensor, float, list):
+    """read video using decord.VideoReader and return also per-frame timestamps"""
+    import decord
+    video_path = ele["video"]
+    st = time.time()
+    vr = decord.VideoReader(video_path)
+    if 'video_start' in ele or 'video_end' in ele:
+        raise NotImplementedError("not support start_pts and end_pts in decord for now.")
+    total_frames, video_fps = len(vr), vr.get_avg_fps()
+    logger.info(f"decord:  {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s")
+    nframes = smart_nframes(ele, total_frames=total_frames, video_fps=video_fps)
+    idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
+    start_time = ele.get("video_start", 0.0) # TODO: 
+    timestamps = [start_time + i / video_fps for i in idx]
+    video = vr.get_batch(idx).asnumpy()
+    video = torch.tensor(video).permute(0, 3, 1, 2)  # Convert to TCHW format
+    sample_fps = nframes / max(total_frames, 1e-6) * video_fps
+    return video, sample_fps, timestamps
+
+
+VIDEO_READER_BACKENDS = {
+    "decord": _read_video_decord,
+    "torchvision": _read_video_torchvision,
+}
+
+
+@lru_cache(maxsize=1)
+def get_video_reader_backend() -> str:
+    if is_decord_available():
+        video_reader_backend = "decord"
+    else:
+        video_reader_backend = "torchvision"
+    return video_reader_backend
+
+
+
+
+def fetch_video(ele: dict, return_video_sample_fps: bool = False) -> torch.Tensor | list[Image.Image]:
+    if isinstance(ele["video"], str):
+        video_reader_backend = get_video_reader_backend()
+        try:
+            video, sample_fps, timestamps = VIDEO_READER_BACKENDS[video_reader_backend](ele)
+        except Exception as e:
+            logger.warning(f"video_reader_backend {video_reader_backend} error, use torchvision as default, msg: {e}")
+            video, sample_fps, timestamps = VIDEO_READER_BACKENDS["torchvision"](ele)
+
+        nframes, _, height, width = video.shape
+
+        if return_video_sample_fps:
+            return video, sample_fps, timestamps
+        return video
+    else:
+        assert isinstance(ele["video"], (list, tuple))
+        process_info = ele.copy()
+        process_info.pop("type", None)
+        process_info.pop("video", None)
+        images = [
+            fetch_image({"image": video_element, **process_info})
+            for video_element in ele["video"]
+        ]
+        nframes = adjust_by_factor(len(images), FRAME_FACTOR, method='ceil')
+        if len(images) < nframes:
+            images.extend([images[-1]] * (nframes - len(images)))
+        
+        timestamps = [-1 for i in range(nframes)] # not sure about this
+        if return_video_sample_fps:
+            return images, process_info.pop("fps", 2.0), timestamps
+        return images
+    
+class Eagle2_5_VLProcessorKwargs(ProcessingKwargs, total=False):
+    # see processing_utils.ProcessingKwargs documentation for usage.
+    _defaults = {
+        "text_kwargs": {
+            "padding": False,
+        },
+        "images_kwargs": {},
+        "videos_kwargs": {"max_dynamic_tiles": 1},
+    }
+
+
+class Eagle2_5_VLProcessor(ProcessorMixin):
+    r"""
+    Constructs a Eagle2_5_VL processor which wraps a Eagle2_5_VL video processor, Eagle2_5_VL image processor and a Eagle2_5_VL tokenizer into a single processor.
+
+    [`Eagle2_5_VLProcessor`] offers all the functionalities of [`Eagle2_5_VLVideoProcessor`], [`Eagle2_5_VLImageProcessor`] and [`Eagle2_5_VLTokenizer`]. See the
+    [`~Eagle2_5_VLVideoProcessor.__call__`], [`~Eagle2_5_VLProcessor.__call__`] and [`~Eagle2_5_VLProcessor.decode`] for more information.
+
+    Args:
+        image_processor ([`LlavaOnevisionImageProcessor`], *optional*):
+            The image processor is a required input.
+        tokenizer ([`LlamaTokenizerFast`], *optional*):
+            The tokenizer is a required input.
+        num_image_tokens (`int`, *optional*):
+            Number of image tokens for one imagethat will be returned by vision tower.
+        vision_feature_select_strategy (`str`, *optional*):
+            The feature selection strategy used to select the vision feature from the vision backbone.
+            Shoudl be same as in model's config
+        chat_template (`str`, *optional*): A Jinja template which will be used to convert lists of messages
+            in a chat into a tokenizable string.
+        image_token (`str`, *optional*, defaults to `"<image>"`):
+            Special token used to denote image location.
+        video_token (`str`, *optional*, defaults to `"<video>"`):
+            Special token used to denote video location.
+    """
+
+    attributes = ["image_processor", "tokenizer"]
+    valid_kwargs = [
+        "chat_template",
+        "num_image_tokens",
+        "vision_feature_select_strategy",
+        "image_token",
+        "video_token",
+        "images_kwargs",
+        "videos_kwargs",
+        "text_kwargs",
+    ]
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = "AutoTokenizer"
+
+    def __init__(
+        self,
+        image_processor=None,
+        tokenizer=None,
+        vision_feature_select_strategy=None,
+        chat_template=None,
+        image_token='<IMG_CONTEXT>',
+        video_token='<IMG_CONTEXT>',
+        tokens_per_tile=256,
+        image_placeholder='image',
+        video_placeholder='video',
+        image_start_token='<img>',
+        image_end_token='</img>',
+        **kwargs,
+    ):  
+        self.vision_feature_select_strategy = vision_feature_select_strategy
+        self.image_token = tokenizer.image_token if hasattr(tokenizer, "image_token") else image_token
+        self.video_token = tokenizer.video_token if hasattr(tokenizer, "video_token") else video_token
+        self.image_token_id = (
+            tokenizer.image_token_id
+            if getattr(tokenizer, "image_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.image_token)
+        )
+        self.video_token_id = (
+            tokenizer.video_token_id
+            if getattr(tokenizer, "video_token_id", None)
+            else tokenizer.convert_tokens_to_ids(self.video_token)
+        )
+        self.image_placeholder = image_placeholder
+        self.video_placeholder = video_placeholder
+        self.tokens_per_tile = tokens_per_tile
+        self.image_start_token = image_start_token
+        self.image_end_token = image_end_token
+        if 'auto_map' in kwargs:
+            self.auto_map = kwargs['auto_map']
+        super().__init__(image_processor, tokenizer, chat_template=chat_template)
+
+    
+    def replace_media_placeholder(self, text, image_list, video_list, timestamps_list, fps_list, **output_kwargs):
+
+        num_of_images_in_this_sample = 0
+        num_of_videos_in_this_sample = 0
+        # Regular expression pattern to match formats like <image-1> or <video-2>
+        pattern = re.compile(rf"<({self.image_placeholder}|{self.video_placeholder})-(\d+)>")
+        unified_frame_list = []
+
+        image_min_dynamic_tiles = output_kwargs['images_kwargs'].get("min_dynamic_tiles", self.image_processor.min_dynamic_tiles)
+        image_max_dynamic_tiles = output_kwargs['images_kwargs'].get("max_dynamic_tiles", self.image_processor.max_dynamic_tiles)
+        image_use_thumbnail = output_kwargs['images_kwargs'].get("use_thumbnail", self.image_processor.use_thumbnail)
+        video_min_dynamic_tiles = output_kwargs['videos_kwargs'].get("min_dynamic_tiles", self.image_processor.min_dynamic_tiles)
+        video_max_dynamic_tiles = output_kwargs['videos_kwargs'].get("max_dynamic_tiles", self.image_processor.max_dynamic_tiles)
+        video_use_thumbnail = output_kwargs['videos_kwargs'].get("use_thumbnail", self.image_processor.use_thumbnail)
+        
+        tile_size = self.image_processor.size.get("height", 448)
+        
+
+        # Function to replace tags in a single text
+        def replace_in_text(text):
+            # repl callback function for each match replacement operation
+            def repl(match):
+                nonlocal unified_frame_list
+                nonlocal num_of_images_in_this_sample
+                nonlocal num_of_videos_in_this_sample
+                media_type = match.group(1)          # 'image' or 'video'
+                idx_in_list = int(match.group(2)) - 1   # Convert to list index (0-based)
+                # Select the corresponding path based on media type
+                idx_mapper = {0: "first", 1: "second", 2: "third", 3: "fourth", 4: "fifth", 5: "sixth", 6: "seventh", 7: "eighth", 8: "ninth", 9: "tenth"}  
+                if media_type == 'image':
+                    image_inputs = self.image_processor(images=[image_list[idx_in_list]], videos=None, **output_kwargs["images_kwargs"])
+                    num_all_tiles = image_inputs["pixel_values"].shape[0]
+                    special_placeholder = f"<image {idx_in_list+1}>{self.image_start_token}{self.image_token * num_all_tiles * self.tokens_per_tile}{self.image_end_token}"
+                    unified_frame_list.append(image_inputs)
+                    num_of_images_in_this_sample += 1
+                    
+                elif media_type == 'video':
+                    video_inputs = self.image_processor(images=None, videos=[video_list[idx_in_list]], **output_kwargs["videos_kwargs"])
+                    num_all_tiles = video_inputs["pixel_values"].shape[0]
+                    image_sizes = video_inputs["image_sizes"]
+                    if timestamps_list is not None and -1 not in timestamps_list:
+                        frame_timestamps = timestamps_list[idx_in_list]
+                    else:
+                        frame_timestamps = None
+                    sampled_fps = fps_list[idx_in_list] if fps_list is not None else None
+                    
+                    num_of_tiles_each_frame = [
+                        self.get_number_tiles_based_on_image_size(image_size, video_min_dynamic_tiles, video_max_dynamic_tiles, video_use_thumbnail, tile_size)
+                        for image_size in image_sizes                           
+                    ]
+                    assert sum(num_of_tiles_each_frame) == num_all_tiles, f"The number of tiles in each frame is not equal to the total number of tiles: {sum(num_of_tiles_each_frame)} != {num_all_tiles}"
+                    
+                    if frame_timestamps is not None:
+                        assert len(frame_timestamps) == len(num_of_tiles_each_frame), f"The number of timestamps is not equal to the number of frames: {len(frame_timestamps)} != {len(num_of_tiles_each_frame)}"
+                        special_placeholder = [f"Frame {i+1} sample at {frame_timestamps[i]:.2f}s: {self.image_start_token}{self.image_token * num_of_tiles * self.tokens_per_tile}{self.image_end_token}" for i, num_of_tiles in enumerate(num_of_tiles_each_frame)]
+                    else:
+                        special_placeholder = [f"Frame {i+1}: {self.image_start_token}{self.image_token * num_of_tiles * self.tokens_per_tile}{self.image_end_token}" for i, num_of_tiles in enumerate(num_of_tiles_each_frame)]
+                    
+                    if sampled_fps is not None:
+                        special_placeholder = f"The {idx_mapper[idx_in_list]} video sampled with {sampled_fps:.2f} fps: " + "".join(special_placeholder)
+                    else:
+                        special_placeholder = f"The {idx_mapper[idx_in_list]} video: " + "".join(special_placeholder)
+                    unified_frame_list.append(video_inputs)
+                    num_of_videos_in_this_sample += 1
+                else:
+                    raise ValueError(f'Unknown media type: {media_type}')
+                return special_placeholder
+            return pattern.sub(repl, text)
+        text = replace_in_text(text)
+        if len(unified_frame_list) > 0:
+            pixel_values = torch.cat([frame["pixel_values"] for frame in unified_frame_list])
+            image_sizes = torch.cat([frame["image_sizes"] for frame in unified_frame_list])
+        else:
+            pixel_values = None
+            image_sizes = None
+        return text, pixel_values, image_sizes, num_of_images_in_this_sample, num_of_videos_in_this_sample
+    
+    def __call__(
+        self,
+        images: ImageInput = None,
+        text: Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None,
+        audio=None,
+        videos: VideoInput = None,
+        **kwargs: Unpack[Eagle2_5_VLProcessorKwargs],
+    ) -> BatchFeature:
+        """
+        Main method to prepare for the model one or several sequences(s) and image(s). This method forwards the `text`
+        and `kwargs` arguments to LlamaTokenizerFast's [`~LlamaTokenizerFast.__call__`] if `text` is not `None` to encode
+        the text. To prepare the image(s), this method forwards the `images` and `kwrags` arguments to
+        LlavaNextImageProcessor's [`~LlavaNextImageProcessor.__call__`] if `images` is not `None`. Please refer to the docstring
+        of the above two methods for more information.
+
+        Args:
+            images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
+                The image or batch of images to be prepared. Each image can be a PIL image, NumPy array or PyTorch
+                tensor. Both channels-first and channels-last formats are supported.
+            text (`str`, `List[str]`, `List[List[str]]`):
+                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
+                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
+                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
+            videos (`np.ndarray`, `torch.Tensor`, `List[np.ndarray]`, `List[torch.Tensor]`):
+                The image or batch of videos to be prepared. Each video can be a 4D NumPy array or PyTorch
+
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+            - **pixel_values_videos** -- Pixel values of a video input to be fed to a model. Returned when `videos` is not `None`.
+            - **image_sizes** -- Size of each image that will be used to unpad an image. Returned when `images` is not `None`.
+        """
+
+
+        output_kwargs = self._merge_kwargs(
+            Eagle2_5_VLProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        
+        if isinstance(text, str):
+            text_list = [text]
+        elif not isinstance(text, list) and not isinstance(text[0], str):
+            raise ValueError("Invalid input text. Please provide a string, or a list of strings")
+        elif isinstance(text, list) and isinstance(text[0], str):
+            text_list = text
+        
+        if images is None: images = []
+        if videos is None: videos = []
+        
+        pixel_values_list = []
+        image_sizes_list = []
+        new_sample_list = []
+        image_start_idx = 0
+        video_start_idx = 0
+        timestamps_batch = output_kwargs['videos_kwargs'].pop("timestamps", None)
+        fps_batch = output_kwargs['videos_kwargs'].pop("fps", None)
+        for sample in text_list:
+            timestamps_list = timestamps_batch[video_start_idx:] if timestamps_batch is not None else None
+            fps_list = fps_batch[video_start_idx:] if fps_batch is not None else None
+            sample, pixel_values, image_sizes, num_of_images_in_this_sample, num_of_videos_in_this_sample = self.replace_media_placeholder(sample, images[image_start_idx:], videos[video_start_idx:], timestamps_list, fps_list, **output_kwargs)
+            new_sample_list.append(sample)
+            if pixel_values is not None:
+                pixel_values_list.append(pixel_values)
+                image_sizes_list.append(image_sizes)
+            image_start_idx += num_of_images_in_this_sample
+            video_start_idx += num_of_videos_in_this_sample
+
+        if len(pixel_values_list) > 0:
+            image_inputs = {"pixel_values": torch.cat(pixel_values_list), "image_sizes":  torch.cat(image_sizes_list)}
+        else:
+            image_inputs = {}
+        video_inputs = {}
+        text_inputs = self.tokenizer(new_sample_list, **output_kwargs["text_kwargs"])
+        return BatchFeature(data={**text_inputs, **image_inputs, **video_inputs})
+
+    def get_number_tiles_based_on_image_size(self, image_size: tuple, min_num: int, max_num: int, use_thumbnail: bool, tile_size: int) -> int:
+        """
+        Get the number of tiles based on the image size.
+        """
+        orig_height, orig_width = image_size
+        aspect_ratio = orig_width / orig_height
+        # calculate the existing image aspect ratio
+        target_ratios = set(
+            (i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if
+            i * j <= max_num and i * j >= min_num)
+        target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+
+        # find the closest aspect ratio to the target
+        target_aspect_ratio = self.image_processor.find_closest_aspect_ratio(
+            aspect_ratio, target_ratios, orig_width, orig_height, tile_size)
+        tiles_num = target_aspect_ratio[0] * target_aspect_ratio[1]
+        if use_thumbnail and tiles_num > 1:
+            tiles_num += 1
+        return tiles_num
+
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode with CLIP->Llama
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode with CLIP->Llama
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to LlamaTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    @property
+    # Copied from transformers.models.clip.processing_clip.CLIPProcessor.model_input_names
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))
+
+    # override to save video-config in a separate config file
+    def save_pretrained(self, save_directory, **kwargs):
+        if os.path.isfile(save_directory):
+            raise ValueError(f"Provided path ({save_directory}) should be a directory, not a file")
+        os.makedirs(save_directory, exist_ok=True)
+
+        outputs = super().save_pretrained(save_directory, **kwargs)
+        return outputs
+
+    # override to load video-config from a separate config file
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        processor = super().from_pretrained(pretrained_model_name_or_path, **kwargs)
+
+        # if return_unused_kwargs a tuple is returned where the second element is 'unused_kwargs'
+        if isinstance(processor, tuple):
+            processor = processor[0]
+        return processor
+
+    # Copy from https://github.com/QwenLM/Qwen2.5-VL/blob/main/qwen-vl-utils/src/qwen_vl_utils/vision_process.py
+    def process_vision_info(
+        self,
+        conversations: list[dict] | list[list[dict]],
+        return_video_kwargs: bool = False,
+    ) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]] | None, Optional[dict]]:
+
+        vision_infos = self.extract_vision_info(conversations)
+        ## Read images or videos
+        image_inputs = []
+        video_inputs = []
+        video_sample_fps_list = []
+        video_timestamps_list = []
+        for vision_info in vision_infos:
+            if "image" in vision_info or "image_url" in vision_info:
+                image_inputs.append(fetch_image(vision_info))
+            elif "video" in vision_info:
+                video_input, video_sample_fps, video_timestamps = fetch_video(vision_info, return_video_sample_fps=True)
+                video_sample_fps_list.append(video_sample_fps)
+                video_inputs.append(video_input)
+                video_timestamps_list.append(video_timestamps)
+            else:
+                raise ValueError("image, image_url or video should in content.")
+        if len(image_inputs) == 0:
+            image_inputs = None
+        if len(video_inputs) == 0:
+            video_inputs = None
+        if return_video_kwargs:
+            return image_inputs, video_inputs, {'fps': video_sample_fps_list, 'timestamps': video_timestamps_list}
+        return image_inputs, video_inputs
+
+    def extract_vision_info(self, conversations: list[dict] | list[list[dict]]) -> list[dict]:
+        vision_infos = []
+        if isinstance(conversations[0], dict):
+            conversations = [conversations]
+        for conversation in conversations:
+            for message in conversation:
+                if isinstance(message["content"], list):
+                    for ele in message["content"]:
+                        if (
+                            "image" in ele
+                            or "image_url" in ele
+                            or "video" in ele
+                            or ele["type"] in ("image", "image_url", "video")
+                        ):
+                            vision_infos.append(ele)
+        return vision_infos
+    
+    def py_apply_chat_template(self, messages, tokenize=False, add_generation_prompt=False):
+        """
+        Renders a chat conversation using a custom template with verification of tokens.
+
+        The purpose is to check for the existence of tokens like "<image-1>" or "<video-1>" 
+        in the message text and skip adding them if they already exist.
+
+        Args:
+            messages (list): A list of message dictionaries. Each message should contain:
+                - 'role': The role of the speaker (e.g., 'system', 'user', 'assistant').
+                - 'content': Either a string or a list of content blocks. In the list each block may contain:
+                      * 'type': The type of content, such as 'image' or 'video'.
+                      * 'text': The actual text if present.
+                      * Other keys such as 'image', 'image_url', or 'video'.
+            add_generation_prompt (bool): If True, appends "<|im_start|>assistant" at the end of the rendered string.
+            tokenize (bool): If True, tokenize the rendered string.
+        Returns:
+            str: The final rendered chat string according to the specified template.
+        """
+        assert tokenize == False, "tokenize is not supported yet"
+        result = ""
+        image_count = 0
+        video_count = 0
+        
+        message_text = ""
+        for idx, message in enumerate(messages):
+            if message.get('role') != 'user': continue
+            # If content is a string, simply output it.
+            content = message.get('content')
+            if isinstance(content, str):
+                message_text += content
+            elif isinstance(content, list):
+                # Process each content item.
+                for item in content:
+                    # If the block is a dictionary and contains text, add it to message_text.
+                    if isinstance(item, dict) and "text" in item:
+                        message_text += item["text"]
+                    # If an item is already a string in the list, add it directly.
+                    elif isinstance(item, str):
+                        message_text += item
+                        
+        for idx, message in enumerate(messages):
+            # If the first message is not from the system, prepend a default system message.
+            if idx == 0 and message.get('role') != 'system':
+                result += "<|im_start|>system\n"
+                result += "You are a helpful assistant.\n"
+                result += "<|im_end|>\n"
+
+            # Start the current message block with its role.
+            result += f"<|im_start|>{message.get('role', '')}\n"
+            content = message.get('content')
+
+            # If content is a string, simply output it.
+            if isinstance(content, str):
+                result += content
+                result += "<|im_end|>\n"
+            else:
+                # Process each content item.
+                for item in content:
+                    # Check if the item is an image (explicitly by type or by key presence).
+                    if (isinstance(item, dict) and (item.get('type') == 'image' or 'image' in item or 'image_url' in item)):
+                        image_count += 1
+                        candidate_token = f"<image-{image_count}>"
+                        # Only add the token if it is not already present in the collected text.
+                        if candidate_token not in message_text:
+                            result += candidate_token
+                    # Check if the item is a video.
+                    elif (isinstance(item, dict) and (item.get('type') == 'video' or 'video' in item)):
+                        video_count += 1
+                        candidate_token = f"<video-{video_count}>"
+                        # Only add the token if it is not already present.
+                        if candidate_token not in message_text:
+                            result += candidate_token
+                    # If the item contains text, add it.
+                    elif isinstance(item, dict) and 'text' in item:
+                        result += item['text']
+                    # If the item is a string (and not handled already), add it.
+                    elif isinstance(item, str):
+                        result += item
+                result += "<|im_end|>\n"
+
+        # Optionally add assistant generation prompt at the end.
+        if add_generation_prompt:
+            result += "<|im_start|>assistant\n"
+
+        return result
+
+
+    @classmethod
+    def from_args_and_dict(cls, args, processor_dict: dict[str, Any], **kwargs):
+        """
+        Instantiates a type of [`~processing_utils.ProcessingMixin`] from a Python dictionary of parameters.
+
+        Args:
+            processor_dict (`Dict[str, Any]`):
+                Dictionary that will be used to instantiate the processor object. Such a dictionary can be
+                retrieved from a pretrained checkpoint by leveraging the
+                [`~processing_utils.ProcessingMixin.to_dict`] method.
+            kwargs (`Dict[str, Any]`):
+                Additional parameters from which to initialize the processor object.
+
+        Returns:
+            [`~processing_utils.ProcessingMixin`]: The processor object instantiated from those
+            parameters.
+        """
+        processor_dict = processor_dict.copy()
+        return_unused_kwargs = kwargs.pop("return_unused_kwargs", False)
+
+        # We have to pop up some unused (but specific) kwargs and then validate that it doesn't contain unused kwargs
+        # If we don't pop, some specific kwargs will raise a warning
+        if "processor_class" in processor_dict:
+            del processor_dict["processor_class"]
+
+        #if "auto_map" in processor_dict:
+        #    del processor_dict["auto_map"]
+
+        unused_kwargs = cls.validate_init_kwargs(processor_config=processor_dict, valid_kwargs=cls.valid_kwargs)
+        processor = cls(*args, **processor_dict)
+
+        # Update processor with kwargs if needed
+        for key in set(kwargs.keys()):
+            if hasattr(processor, key):
+                setattr(processor, key, kwargs.pop(key))
+
+        kwargs.update(unused_kwargs)
+        logger.info(f"Processor {processor}")
+        if return_unused_kwargs:
+            return processor, kwargs
+        else:
+            return processor
+        
+        
+__all__ = ["Eagle2_5_VLProcessor"]

processor_config.json

@@ -0,0 +1,15 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_eagle2_5_vl.Eagle2_5_VLProcessor",
+    "AutoImageProcessor": "image_processing_eagle2_5_vl_fast.Eagle2_5_VLImageProcessorFast"
+  },
+  "image_end_token": "</img>",
+  "image_placeholder": "image",
+  "image_start_token": "<img>",
+  "image_token": "<IMG_CONTEXT>",
+  "processor_class": "Eagle2_5_VLProcessor",
+  "tokens_per_tile": 256,
+  "video_placeholder": "video",
+  "video_token": "<IMG_CONTEXT>",
+  "vision_feature_select_strategy": null
+}