Jofthomas commited on
Commit
3c4371f
·
verified ·
1 Parent(s): 9cde2c4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +100 -40
app.py CHANGED
@@ -2,7 +2,7 @@ import os
2
  import gradio as gr
3
  import requests
4
  import inspect # To get source code for __repr__
5
- import pandas as pd
6
 
7
  # --- Constants ---
8
  DEFAULT_API_URL = "https://jofthomas-unit4-scoring.hf.space/" # Default URL for your FastAPI app
@@ -30,16 +30,24 @@ class BasicAgent:
30
  print(f"Agent returning fixed answer: {fixed_answer}")
31
  return fixed_answer
32
 
 
 
 
33
  def __repr__(self) -> str:
34
  """
35
  Return the source code required to reconstruct this agent.
 
36
  """
37
  imports = [
38
- "import inspect\n" # May not be strictly needed by the agent logic itself
39
  ]
40
- class_source = inspect.getsource(BasicAgent)
41
- full_source = "\n".join(imports) + "\n" + class_source
42
- return full_source
 
 
 
 
43
 
44
  # --- Gradio UI and Logic ---
45
  def get_current_script_content() -> str:
@@ -51,8 +59,9 @@ def get_current_script_content() -> str:
51
  with open(script_path, 'r', encoding='utf-8') as f:
52
  return f.read()
53
  except NameError:
54
- # __file__ is not defined (e.g., running in an interactive interpreter)
55
- print("Warning: __file__ is not defined. Cannot read script content.")
 
56
  return "# Agent code unavailable: __file__ not defined"
57
  except FileNotFoundError:
58
  print(f"Warning: Script file '{script_path}' not found.")
@@ -67,12 +76,28 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
67
  Fetches all questions, runs the BasicAgent on them, submits all answers,
68
  and displays the results.
69
  """
70
-
 
 
 
 
 
 
 
 
 
 
 
 
71
  if profile:
72
  username= f"{profile.username}"
73
-
74
  else:
75
- return "Please Login to Hugging Face with the button.", None
 
 
 
 
76
 
77
  api_url = DEFAULT_API_URL
78
  questions_url = f"{api_url}/questions"
@@ -81,33 +106,46 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
81
  # 1. Instantiate the Agent
82
  try:
83
  agent = BasicAgent()
84
- agent_code = agent.__repr__()
85
- # print(f"Agent Code (first 200): {agent_code[:200]}...") # Debug
 
86
  except Exception as e:
87
- print(f"Error instantiating agent or getting repr: {e}")
88
  return f"Error initializing agent: {e}", None
89
- agent_code=get_current_script_content()
 
 
 
 
 
 
 
90
  # 2. Fetch All Questions
91
  print(f"Fetching questions from: {questions_url}")
92
  try:
93
  response = requests.get(questions_url, timeout=15)
94
- response.raise_for_status()
95
  questions_data = response.json()
96
  if not questions_data:
97
- return "Fetched questions list is empty.", None
 
98
  print(f"Fetched {len(questions_data)} questions.")
99
- status_update = f"Fetched {len(questions_data)} questions. Running agent..."
100
- # Yield intermediate status if using gr.update
101
  except requests.exceptions.RequestException as e:
102
  print(f"Error fetching questions: {e}")
103
  return f"Error fetching questions: {e}", None
104
- except Exception as e:
 
 
 
 
105
  print(f"An unexpected error occurred fetching questions: {e}")
106
  return f"An unexpected error occurred fetching questions: {e}", None
107
 
108
  # 3. Run Agent on Each Question
109
  results_log = [] # To store data for the results table
110
  answers_payload = [] # To store data for the submission API
 
111
  for item in questions_data:
112
  task_id = item.get("task_id")
113
  question_text = item.get("question")
@@ -129,31 +167,33 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
129
  })
130
  except Exception as e:
131
  print(f"Error running agent on task {task_id}: {e}")
132
- # Decide how to handle agent errors - skip? submit default?
133
- # Here, we'll just log and potentially skip submission for this task if needed
134
  results_log.append({
135
  "Task ID": task_id,
136
  "Question": question_text,
137
  "Submitted Answer": f"AGENT ERROR: {e}"
138
  })
139
-
 
140
 
141
  if not answers_payload:
 
 
142
  return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
143
 
144
  # 4. Prepare Submission
145
  submission_data = {
146
  "username": username.strip(),
147
- "agent_code": agent_code,
148
  "answers": answers_payload
149
  }
150
- status_update = f"Agent finished. Submitting {len(answers_payload)} answers..."
151
  print(status_update)
152
 
153
  # 5. Submit to Leaderboard
154
  print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
155
  try:
156
- response = requests.post(submit_url, json=submission_data, timeout=45) # Increased timeout
 
157
  response.raise_for_status()
158
  result_data = response.json()
159
 
@@ -161,31 +201,38 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
161
  final_status = (
162
  f"Submission Successful!\n"
163
  f"User: {result_data.get('username')}\n"
164
- f"Overall Score: {result_data.get('score')}% "
165
- f"({result_data.get('correct_count')}/{result_data.get('total_attempted')} correct)\n"
166
- f"Message: {result_data.get('message')}"
167
  )
168
  print("Submission successful.")
169
  results_df = pd.DataFrame(results_log)
170
  return final_status, results_df
171
 
172
  except requests.exceptions.HTTPError as e:
173
- error_detail = e.response.text
174
  try:
 
175
  error_json = e.response.json()
176
- error_detail = error_json.get('detail', error_detail)
177
  except requests.exceptions.JSONDecodeError:
178
- pass
179
- status_message = f"Submission Failed (HTTP {e.response.status_code}): {error_detail}"
 
180
  print(status_message)
181
  results_df = pd.DataFrame(results_log) # Show attempts even if submission failed
182
  return status_message, results_df
 
 
 
 
 
183
  except requests.exceptions.RequestException as e:
184
  status_message = f"Submission Failed: Network error - {e}"
185
  print(status_message)
186
  results_df = pd.DataFrame(results_log)
187
  return status_message, results_df
188
- except Exception as e:
189
  status_message = f"An unexpected error occurred during submission: {e}"
190
  print(status_message)
191
  results_df = pd.DataFrame(results_log)
@@ -196,25 +243,38 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
196
  with gr.Blocks() as demo:
197
  gr.Markdown("# Basic Agent Evaluation Runner")
198
  gr.Markdown(
199
- "Please clone this space, then modify the code to what you deem relevant."
200
- "Connect to your Hugging Face account using the log in button in the space to use your username, then click Run. "
201
- "This will fetch all questions, run the *very basic* agent on them, "
202
- "submit all answers at once, and display the results."
203
  )
204
 
205
  gr.LoginButton()
206
 
207
  run_button = gr.Button("Run Evaluation & Submit All Answers")
208
 
209
- status_output = gr.Textbox(label="Run Status / Submission Result", lines=4, interactive=False)
210
- results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True)
211
 
212
  # --- Component Interaction ---
 
213
  run_button.click(
214
  fn=run_and_submit_all,
 
215
  outputs=[status_output, results_table]
216
  )
217
 
218
  if __name__ == "__main__":
 
 
 
 
 
 
 
 
 
 
 
219
  print("Launching Gradio Interface for Basic Agent Evaluation...")
220
- demo.launch(debug=True)
 
 
2
  import gradio as gr
3
  import requests
4
  import inspect # To get source code for __repr__
5
+ import pandas as pd
6
 
7
  # --- Constants ---
8
  DEFAULT_API_URL = "https://jofthomas-unit4-scoring.hf.space/" # Default URL for your FastAPI app
 
30
  print(f"Agent returning fixed answer: {fixed_answer}")
31
  return fixed_answer
32
 
33
+ # __repr__ seems intended to get the *source* code, not just representation
34
+ # Let's keep it but note that get_current_script_content might be more robust
35
+ # if the class definition changes significantly or relies on external state.
36
  def __repr__(self) -> str:
37
  """
38
  Return the source code required to reconstruct this agent.
39
+ NOTE: This might be brittle. Using get_current_script_content is likely safer.
40
  """
41
  imports = [
42
+ "import inspect\n"
43
  ]
44
+ try:
45
+ class_source = inspect.getsource(BasicAgent)
46
+ full_source = "\n".join(imports) + "\n" + class_source
47
+ return full_source
48
+ except Exception as e:
49
+ print(f"Error getting source code via inspect: {e}")
50
+ return f"# Could not get source via inspect: {e}"
51
 
52
  # --- Gradio UI and Logic ---
53
  def get_current_script_content() -> str:
 
59
  with open(script_path, 'r', encoding='utf-8') as f:
60
  return f.read()
61
  except NameError:
62
+ # __file__ is not defined (e.g., running in an interactive interpreter or frozen app)
63
+ print("Warning: __file__ is not defined. Cannot read script content this way.")
64
+ # Fallback or alternative method could be added here if needed
65
  return "# Agent code unavailable: __file__ not defined"
66
  except FileNotFoundError:
67
  print(f"Warning: Script file '{script_path}' not found.")
 
76
  Fetches all questions, runs the BasicAgent on them, submits all answers,
77
  and displays the results.
78
  """
79
+ # --- Determine HF Space URL and Print Environment Info ---
80
+ space_host = os.getenv("SPACE_HOST")
81
+ hf_space_url = "Runtime: Locally or unknown environment (SPACE_HOST env var not found)"
82
+ if space_host:
83
+ # Construct the standard URL format for HF Spaces
84
+ hf_space_url = f"Runtime: Hugging Face Space (https://{space_host}.hf.space)"
85
+
86
+ # Print runtime info at the start of the function execution
87
+ print("\n" + "="*60)
88
+ print("Executing run_and_submit_all function...")
89
+ print(hf_space_url) # Print the determined runtime URL
90
+ # --- End Environment Info ---
91
+
92
  if profile:
93
  username= f"{profile.username}"
94
+ print(f"User logged in: {username}")
95
  else:
96
+ print("User not logged in.")
97
+ print("="*60 + "\n") # Close the separator block
98
+ return "Please Login to Hugging Face with the button.", None # Return early
99
+
100
+ print("="*60 + "\n") # Separator after initial checks if logged in
101
 
102
  api_url = DEFAULT_API_URL
103
  questions_url = f"{api_url}/questions"
 
106
  # 1. Instantiate the Agent
107
  try:
108
  agent = BasicAgent()
109
+ # Using get_current_script_content() is likely more reliable for submission
110
+ # agent_code = agent.__repr__() # Keep if needed, but prefer file content
111
+ # print(f"Agent Code via __repr__ (first 200): {agent_code[:200]}...") # Debug
112
  except Exception as e:
113
+ print(f"Error instantiating agent: {e}")
114
  return f"Error initializing agent: {e}", None
115
+
116
+ # Get agent code by reading the current script file - generally more robust
117
+ agent_code = get_current_script_content()
118
+ if agent_code.startswith("# Agent code unavailable"):
119
+ print("Warning: Using potentially incomplete agent code due to reading error.")
120
+ # Optional: Fall back to agent.__repr__() if needed
121
+ # agent_code = agent.__repr__()
122
+
123
  # 2. Fetch All Questions
124
  print(f"Fetching questions from: {questions_url}")
125
  try:
126
  response = requests.get(questions_url, timeout=15)
127
+ response.raise_for_status() # Raise HTTPError for bad responses (4xx or 5xx)
128
  questions_data = response.json()
129
  if not questions_data:
130
+ print("Fetched questions list is empty.")
131
+ return "Fetched questions list is empty or invalid format.", None
132
  print(f"Fetched {len(questions_data)} questions.")
133
+ # status_update = f"Fetched {len(questions_data)} questions. Running agent..." # For yield/streaming
 
134
  except requests.exceptions.RequestException as e:
135
  print(f"Error fetching questions: {e}")
136
  return f"Error fetching questions: {e}", None
137
+ except requests.exceptions.JSONDecodeError as e:
138
+ print(f"Error decoding JSON response from questions endpoint: {e}")
139
+ print(f"Response text: {response.text[:500]}") # Log response text for debugging
140
+ return f"Error decoding server response for questions: {e}", None
141
+ except Exception as e: # Catch other potential errors
142
  print(f"An unexpected error occurred fetching questions: {e}")
143
  return f"An unexpected error occurred fetching questions: {e}", None
144
 
145
  # 3. Run Agent on Each Question
146
  results_log = [] # To store data for the results table
147
  answers_payload = [] # To store data for the submission API
148
+ print(f"Running agent on {len(questions_data)} questions...")
149
  for item in questions_data:
150
  task_id = item.get("task_id")
151
  question_text = item.get("question")
 
167
  })
168
  except Exception as e:
169
  print(f"Error running agent on task {task_id}: {e}")
 
 
170
  results_log.append({
171
  "Task ID": task_id,
172
  "Question": question_text,
173
  "Submitted Answer": f"AGENT ERROR: {e}"
174
  })
175
+ # Decide if you want to submit agent errors or skip:
176
+ # answers_payload.append({"task_id": task_id, "submitted_answer": f"AGENT ERROR: {e}"})
177
 
178
  if not answers_payload:
179
+ print("Agent did not produce any answers to submit.")
180
+ # Still show results log even if nothing submitted
181
  return "Agent did not produce any answers to submit.", pd.DataFrame(results_log)
182
 
183
  # 4. Prepare Submission
184
  submission_data = {
185
  "username": username.strip(),
186
+ "agent_code": agent_code, # Using the code read from file
187
  "answers": answers_payload
188
  }
189
+ status_update = f"Agent finished. Submitting {len(answers_payload)} answers for user '{username}'..."
190
  print(status_update)
191
 
192
  # 5. Submit to Leaderboard
193
  print(f"Submitting {len(answers_payload)} answers to: {submit_url}")
194
  try:
195
+ # Ensure submission_data is serializable, agent_code should be string
196
+ response = requests.post(submit_url, json=submission_data, timeout=60) # Increased timeout further
197
  response.raise_for_status()
198
  result_data = response.json()
199
 
 
201
  final_status = (
202
  f"Submission Successful!\n"
203
  f"User: {result_data.get('username')}\n"
204
+ f"Overall Score: {result_data.get('score', 'N/A')}% "
205
+ f"({result_data.get('correct_count', '?')}/{result_data.get('total_attempted', '?')} correct)\n"
206
+ f"Message: {result_data.get('message', 'No message received.')}"
207
  )
208
  print("Submission successful.")
209
  results_df = pd.DataFrame(results_log)
210
  return final_status, results_df
211
 
212
  except requests.exceptions.HTTPError as e:
213
+ error_detail = f"Server responded with status {e.response.status_code}."
214
  try:
215
+ # Try to get more specific error detail from JSON response body
216
  error_json = e.response.json()
217
+ error_detail += f" Detail: {error_json.get('detail', e.response.text)}"
218
  except requests.exceptions.JSONDecodeError:
219
+ # If response is not JSON, use the raw text
220
+ error_detail += f" Response: {e.response.text[:500]}" # Limit length
221
+ status_message = f"Submission Failed: {error_detail}"
222
  print(status_message)
223
  results_df = pd.DataFrame(results_log) # Show attempts even if submission failed
224
  return status_message, results_df
225
+ except requests.exceptions.Timeout:
226
+ status_message = "Submission Failed: The request timed out."
227
+ print(status_message)
228
+ results_df = pd.DataFrame(results_log)
229
+ return status_message, results_df
230
  except requests.exceptions.RequestException as e:
231
  status_message = f"Submission Failed: Network error - {e}"
232
  print(status_message)
233
  results_df = pd.DataFrame(results_log)
234
  return status_message, results_df
235
+ except Exception as e: # Catch unexpected errors during submission phase
236
  status_message = f"An unexpected error occurred during submission: {e}"
237
  print(status_message)
238
  results_df = pd.DataFrame(results_log)
 
243
  with gr.Blocks() as demo:
244
  gr.Markdown("# Basic Agent Evaluation Runner")
245
  gr.Markdown(
246
+ "Please clone this space, then modify the code to define your agent's logic within the `BasicAgent` class. " # Clarified instructions
247
+ "Log in to your Hugging Face account using the button below. This uses your HF username for submission. "
248
+ "Click 'Run Evaluation & Submit All Answers' to fetch questions, run your agent, submit answers, and see the score."
 
249
  )
250
 
251
  gr.LoginButton()
252
 
253
  run_button = gr.Button("Run Evaluation & Submit All Answers")
254
 
255
+ status_output = gr.Textbox(label="Run Status / Submission Result", lines=5, interactive=False) # Increased lines
256
+ results_table = gr.DataFrame(label="Questions and Agent Answers", wrap=True, max_rows=10) # Added max_rows
257
 
258
  # --- Component Interaction ---
259
+ # Use the profile information directly from the LoginButton state (implicitly passed)
260
  run_button.click(
261
  fn=run_and_submit_all,
262
+ # Input is implicitly the profile data from LoginButton state
263
  outputs=[status_output, results_table]
264
  )
265
 
266
  if __name__ == "__main__":
267
+ print("\n" + "-"*30 + " App Starting " + "-"*30)
268
+ # Check for SPACE_HOST at startup for information
269
+ space_host_startup = os.getenv("SPACE_HOST")
270
+ if space_host_startup:
271
+ print(f"✅ SPACE_HOST found: {space_host_startup}")
272
+ print(f" App should be available at: https://{space_host_startup}.hf.space")
273
+ else:
274
+ print("ℹ️ SPACE_HOST environment variable not found (running locally or not on standard HF Space runtime).")
275
+ print(" App will likely be available at local URLs printed by Gradio below.")
276
+ print("-"*(60 + len(" App Starting ")) + "\n")
277
+
278
  print("Launching Gradio Interface for Basic Agent Evaluation...")
279
+ # Set share=False as the primary access point is the HF Space URL
280
+ demo.launch(debug=True, share=False)