|
|
|
import os |
|
import gradio as gr |
|
import requests |
|
import openai |
|
from smolagents import OpenAIServerModel, DuckDuckGoSearchTool, CodeAgent, WikipediaSearchTool |
|
from pathlib import Path |
|
import tempfile |
|
from smolagents.tools import PipelineTool, Tool |
|
import pathlib |
|
from typing import Union, Optional |
|
import pandas as pd |
|
from tabulate import tabulate |
|
import re |
|
|
|
|
|
|
|
DEFAULT_API_URL = "https://agents-course-unit4-scoring.hf.space" |
|
|
|
class SpeechToTextTool(PipelineTool): |
|
""" |
|
Transcribes an audio file to text using the OpenAI Whisper API. |
|
Only local file paths are supported. |
|
""" |
|
default_checkpoint = "openai/whisper-1" |
|
description = ( |
|
"This tool sends an audio file to OpenAI Whisper and returns the " |
|
"transcribed text." |
|
) |
|
name = "transcriber" |
|
inputs = { |
|
"audio": { |
|
"type": "string", |
|
"description": "Absolute or relative path to a local audio file.", |
|
} |
|
} |
|
output_type = "string" |
|
|
|
|
|
|
|
|
|
def __call__(self, audio: str) -> str: |
|
""" |
|
Convenience wrapper so the tool can be used like a regular function: |
|
text = SpeechToTextTool()(path_to_audio) |
|
""" |
|
return self._transcribe(audio) |
|
|
|
|
|
|
|
|
|
@staticmethod |
|
def _transcribe(audio_path: str) -> str: |
|
|
|
if not isinstance(audio_path, str): |
|
raise TypeError( |
|
"Parameter 'audio' must be a string containing the file path." |
|
) |
|
path = Path(audio_path).expanduser().resolve() |
|
if not path.is_file(): |
|
raise FileNotFoundError(f"No such audio file: {path}") |
|
|
|
|
|
with path.open("rb") as fp: |
|
response = openai.audio.transcriptions.create( |
|
file=fp, |
|
model="whisper-1", |
|
response_format="text" |
|
) |
|
|
|
|
|
return response |
|
|
|
class ExcelToTextTool(Tool): |
|
"""Render an Excel worksheet as Markdown text.""" |
|
|
|
|
|
|
|
|
|
name = "excel_to_text" |
|
description = ( |
|
"Read an Excel file and return a Markdown table of the requested sheet. " |
|
"Accepts either the sheet name or the zero-based index." |
|
) |
|
|
|
inputs = { |
|
"excel_path": { |
|
"type": "string", |
|
"description": "Path to the Excel file (.xlsx / .xls).", |
|
}, |
|
"sheet_name": { |
|
"type": "string", |
|
"description": ( |
|
"Worksheet name or zeroβbased index *as a string* (optional; default first sheet)." |
|
), |
|
"nullable": True, |
|
}, |
|
} |
|
|
|
output_type = "string" |
|
|
|
|
|
|
|
|
|
def forward( |
|
self, |
|
excel_path: str, |
|
sheet_name: Optional[str] = None, |
|
) -> str: |
|
"""Load *excel_path* and return the sheet as a Markdown table.""" |
|
|
|
path = pathlib.Path(excel_path).expanduser().resolve() |
|
if not path.exists(): |
|
return f"Error: Excel file not found at {path}" |
|
|
|
try: |
|
|
|
sheet: Union[str, int] |
|
if sheet_name is None or sheet_name == "": |
|
sheet = 0 |
|
else: |
|
|
|
sheet = int(sheet_name) if sheet_name.isdigit() else sheet_name |
|
|
|
|
|
df = pd.read_excel(path, sheet_name=sheet) |
|
|
|
|
|
if hasattr(pd.DataFrame, "to_markdown"): |
|
return df.to_markdown(index=False) |
|
from tabulate import tabulate |
|
|
|
return tabulate(df, headers="keys", tablefmt="github", showindex=False) |
|
|
|
except Exception as exc: |
|
return f"Error reading Excel file: {exc}" |
|
|
|
|
|
def download_file_if_any(base_api_url: str, task_id: str) -> str | None: |
|
""" |
|
Try GET /files/{task_id}. |
|
β’ On HTTP 200 β save to a temp dir and return local path. |
|
β’ On 404 β return None. |
|
β’ On other errors β raise so caller can log / handle. |
|
""" |
|
url = f"{base_api_url}/files/{task_id}" |
|
try: |
|
resp = requests.get(url, timeout=30) |
|
if resp.status_code == 404: |
|
return None |
|
resp.raise_for_status() |
|
except requests.exceptions.HTTPError as e: |
|
|
|
raise e |
|
|
|
|
|
|
|
cdisp = resp.headers.get("content-disposition", "") |
|
filename = task_id |
|
if "filename=" in cdisp: |
|
m = re.search(r'filename="([^"]+)"', cdisp) |
|
if m: |
|
filename = m.group(1) |
|
|
|
tmp_dir = Path(tempfile.gettempdir()) / "gaia_files" |
|
tmp_dir.mkdir(exist_ok=True) |
|
file_path = tmp_dir / filename |
|
with open(file_path, "wb") as f: |
|
f.write(resp.content) |
|
return str(file_path) |
|
|
|
|
|
|
|
class BasicAgent: |
|
def __init__(self): |
|
self.agent = CodeAgent( |
|
model=OpenAIServerModel(model_id="gpt-4o"), |
|
tools=[DuckDuckGoSearchTool(), WikipediaSearchTool(), SpeechToTextTool(), ExcelToTextTool()], |
|
add_base_tools=True, |
|
additional_authorized_imports=['pandas','numpy','csv','subprocess'] |
|
) |
|
|
|
print("BasicAgent initialized.") |
|
|
|
def __call__(self, question: str) -> str: |
|
print(f"Agent received question (first 50 chars): {question[:50]}...") |
|
fixed_answer = self.agent.run(question) |
|
print(f"Agent returning answer: {fixed_answer}") |
|
return fixed_answer |
|
|
|
def run_and_submit_all( profile: gr.OAuthProfile | None): |
|
""" |
|
Fetches all questions, runs the BasicAgent on them, submits all answers, |
|
and displays the results. |
|
""" |
|
|
|
space_id = "innovation64/Final_Assignment_codeagent" |
|
|
|
if profile: |
|
username= f"{profile.username}" |
|
print(f"User logged in: {username}") |
|
else: |
|
print("User not logged in.") |
|
return "Please Login to Hugging Face with the button.", None |
|
|
|
api_url = DEFAULT_API_URL |
|
questions_url = f"{api_url}/questions" |
|
submit_url = f"{api_url}/submit" |
|
|
|
|
|
try: |
|
agent = BasicAgent() |
|
except Exception as e: |
|
print(f"Error instantiating agent: {e}") |
|
return f"Error initializing agent: {e}", None |
|
|
|
agent_code = f"https://huggingface.co./spaces/{space_id}/tree/main" |
|
print(agent_code) |
|
|
|
|
|
print(f"Fetching questions from: {questions_url}") |
|
try: |
|
response = requests.get(questions_url, timeout=15) |
|
response.raise_for_status() |
|
questions_data = response.json() |
|
if not questions_data: |
|
print("Fetched questions list is empty.") |
|
return "Fetched questions list is empty or invalid format.", None |
|
print(f"Fetched {len(questions_data)} questions.") |
|
except requests.exceptions.RequestException as e: |
|
print(f"Error fetching questions: {e}") |
|
return f"Error fetching questions: {e}", None |
|
except requests.exceptions.JSONDecodeError as e: |
|
print(f"Error decoding JSON response from questions endpoint: {e}") |
|
print(f"Response text: {response.text[:500]}") |
|
return f"Error decoding server response for questions: {e}", None |
|
except Exception as e: |
|
print(f"An unexpected error occurred fetching questions: {e}") |
|
return f"An unexpected error occurred fetching questions: {e}", None |
|
|
|
|
|
results_log = [] |
|
answers_payload = [] |
|
print(f"Running agent on {len(questions_data)} questions...") |
|
for item in questions_data: |
|
task_id = item.get("task_id") |
|
question_text = item.get("question") |
|
|
|
|
|
try: |
|
file_path = download_file_if_any(api_url, task_id) |
|
except Exception as e: |
|
file_path = None |
|
print(f"[file fetch error] {task_id}: {e}") |
|
|
|
|
|
if file_path: |
|
q_for_agent = ( |
|
f"{question_text}\n\n" |
|
f"---\n" |
|
f"A file was downloaded for this task and saved locally at:\n" |
|
f"{file_path}\n" |
|
f"---\n\n" |
|
) |
|
else: |
|
q_for_agent = question_text |
|
|
|
if not task_id or question_text is None: |
|
print(f"Skipping item with missing task_id or question: {item}") |
|
continue |
|
try: |
|
submitted_answer = agent(q_for_agent) |
|
answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer}) |
|
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer}) |
|
except Exception as e: |
|
print(f"Error running agent on task {task_id}: {e}") |
|
results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"}) |
|
|
|
if not answers_payload: |
|
print("Agent did not produce any answers to submit.") |
|
return "Agent did not produce any answers to submit.", pd.DataFrame(results_log) |
|
|
|
|
|
submission_data = {"username": username.strip(), "agent_code": agent_code, "answers": answers_payload} |
|
status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..." |
|
print(status_update) |
|
|
|
|
|
print(f"Submitting {len(answers_payload)} answers to: {submit_url}") |
|
try: |
|
response = requests.post(submit_url, json=submission_data, timeout=60) |
|
response.raise_for_status() |
|
result_data = response.json() |
|
final_status = ( |
|
f"Submission Successful!\n" |
|
f"User: {result_data.get('username')}\n" |
|
f"Overall Score: {result_data.get('score', 'N/A')}% " |
|
f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n" |
|
f"Message: {result_data.get('message', 'No message received.')}" |
|
) |
|
print("Submission successful.") |
|
results_df = pd.DataFrame(results_log) |
|
return final_status, results_df |
|
except requests.exceptions.HTTPError as e: |
|
error_detail = f"Server responded with status {e.response.status_code}." |
|
try: |
|
error_json = e.response.json() |
|
error_detail += f" Detail: {error_json.get('detail', e.response.text)}" |
|
except requests.exceptions.JSONDecodeError: |
|
error_detail += f" Response: {e.response.text[:500]}" |
|
status_message = f"Submission Failed: {error_detail}" |
|
print(status_message) |
|
results_df = pd.DataFrame(results_log) |
|
return status_message, results_df |
|
except requests.exceptions.Timeout: |
|
status_message = "Submission Failed: The request timed out." |
|
print(status_message) |
|
results_df = pd.DataFrame(results_log) |
|
return status_message, results_df |
|
except requests.exceptions.RequestException as e: |
|
status_message = f"Submission Failed: Network error - {e}" |
|
print(status_message) |
|
results_df = pd.DataFrame(results_log) |
|
return status_message, results_df |
|
except Exception as e: |
|
status_message = f"An unexpected error occurred during submission: {e}" |
|
print(status_message) |
|
results_df = pd.DataFrame(results_log) |
|
return status_message, results_df |
|
|
|
|
|
|
|
with gr.Blocks() as demo: |
|
gr.Markdown("# Basic Agent Evaluation Runner") |
|
gr.Markdown( |
|
""" |
|
**Instructions:** |
|
1. Please clone this space, then modify the code to define your agent's logic, the tools, the necessary packages, etc ... |
|
2. Log in to your Hugging Face account using the button below. This uses your HF username for submission. |
|
3. Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score. |
|
--- |
|
**Disclaimers:** |
|
Once clicking on the "submit button, it can take quite some time ( this is the time for the agent to go through all the questions). |
|
This space provides a basic setup and is intentionally sub-optimal to encourage you to develop your own, more robust solution. For instance for the delay process of the submit button, a solution could be to cache the answers and submit in a seperate action or even to answer the questions in async. |
|
""" |
|
) |
|
|
|
gr.LoginButton() |
|
|
|
run_button = gr.Button("Run Evaluation & Submit All Answers") |
|
|
|
status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) |
|
|
|
results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True) |
|
|
|
run_button.click( |
|
fn=run_and_submit_all, |
|
outputs=[status_output, results_table] |
|
) |
|
|
|
if __name__ == "__main__": |
|
print("\n" + "-"*30 + " App Starting " + "-"*30) |
|
|
|
space_host_startup = os.getenv("SPACE_HOST") |
|
space_id_startup = "innovation64/Final_Assignment_codeagent" |
|
|
|
if space_host_startup: |
|
print(f"β
SPACE_HOST found: {space_host_startup}") |
|
print(f" Runtime URL should be: https://{space_host_startup}.hf.space") |
|
else: |
|
print("βΉοΈ SPACE_HOST environment variable not found (running locally?).") |
|
|
|
if space_id_startup: |
|
print(f"β
SPACE_ID found: {space_id_startup}") |
|
print(f" Repo URL: https://huggingface.co./spaces/{space_id_startup}") |
|
print(f" Repo Tree URL: https://huggingface.co./spaces/{space_id_startup}/tree/main") |
|
else: |
|
print("βΉοΈ SPACE_ID environment variable not found (running locally?). Repo URL cannot be determined.") |
|
|
|
print("-"*(60 + len(" App Starting ")) + "\n") |
|
|
|
print("Launching Gradio Interface for Basic Agent Evaluation...") |
|
demo.launch(debug=True, share=False) |