File size: 1,443 Bytes
e83010b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 |
import gradio as gr
from transformers import AutoProcessor, PaliGemmaForConditionalGeneration
import torch
# Load model and processor
model_id = "pyimagesearch/finetuned_paligemma_vqav2_small"
model = PaliGemmaForConditionalGeneration.from_pretrained(model_id)
processor = AutoProcessor.from_pretrained("google/paligemma-3b-pt-224")
# Define inference function
def process_image(image, prompt):
# Process the image and prompt using the processor
inputs = processor(image.convert("RGB"), prompt, return_tensors="pt")
try:
# Generate output from the model
output = model.generate(**inputs, max_new_tokens=20)
# Decode and return the output
decoded_output = processor.decode(output[0], skip_special_tokens=True)
# Return the answer (exclude the prompt part from output)
return decoded_output[len(prompt):]
except IndexError as e:
print(f"IndexError: {e}")
return "An error occurred during processing."
# Define the Gradio interface
inputs = [
gr.Image(type="pil"),
gr.Textbox(label="Prompt", placeholder="Enter your question")
]
outputs = gr.Textbox(label="Answer")
# Create the Gradio app
demo = gr.Interface(fn=process_image, inputs=inputs, outputs=outputs, title="Visual Question Answering with Fine-tuned PaliGemma Model", description="Upload an image and ask questions to get answers.")
# Launch the app
demo.launch() |