Update Screenshot img upload. Allow request interrupt previous prompt. Allow question editable

app.py

@@ -1,60 +1,44 @@
-# Access site: https://binkhoale1812-interview-ai.hf.space/
-import os, tempfile
+# Interview Q&A – FastAPI backend
+import base64, io, json, logging, os, tempfile
 from pathlib import Path
 from typing import Dict
 
-# Server
 from fastapi import FastAPI, File, UploadFile, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse, FileResponse
 from fastapi.staticfiles import StaticFiles
 
-# AI + LLM
-import torch # For transformer
+# AI / LLM
 from google import genai
 from google.genai import types
 
-# Audio Transcribe
+# ASR
+import numpy as np
 from pydub import AudioSegment
 from transformers import WhisperProcessor, WhisperForConditionalGeneration
-import numpy as np
 
-############################################
-# ── Configuration ────────────────────────
-############################################
+# Misc
+from PIL import Image
+
+##############################################################################
 
 GEMINI_API_KEY = os.getenv("GEMINI_API_KEY")
 if not GEMINI_API_KEY:
-    raise RuntimeError("GEMINI_API_KEY environment variable must be set!")
+    raise RuntimeError("❌ GEMINI_API_KEY must be set as env var")
 
-# Tiny Whisper model is light enough for CPU Spaces; change if GPU is available
 ASR_MODEL_ID = "openai/whisper-small.en"
-ASR_LANGUAGE = "en"  # Force to English for interview setting
-SAMPLE_RATE  = 16000
+ASR_LANGUAGE = "en"
+SAMPLE_RATE  = 16_000
 
-############################################
-# ── FastAPI App ───────────────────────────
-############################################
+##############################################################################
 
 app = FastAPI(title="Interview Q&A Assistant", docs_url="/docs")
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=["*"],
-    allow_methods=["*"],
-    allow_headers=["*"],
+    allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
 )
-
-# Serve frontend assets
 app.mount("/statics", StaticFiles(directory="statics"), name="statics")
 
-############################################
-# ── Global objects (lazy‑loaded) ──────────
-############################################
-
-# Globals
-processor = None
-model = None
-
 # Enable Logging for Debugging
 import psutil
 import logging
@@ -84,84 +68,108 @@ def check_system_resources():
         logger.warning("⚠️ High Disk usage detected!")
 check_system_resources()
 
-# Startup
-@app.on_event("startup")
-async def load_models():
-    global processor, model
-    cache = Path("model_cache"); cache.mkdir(exist_ok=True)
-    # in startup (Transformer Whisper processing)
-    processor = WhisperProcessor.from_pretrained(ASR_MODEL_ID, cache_dir=cache)
-    model      = WhisperForConditionalGeneration.from_pretrained(ASR_MODEL_ID, cache_dir=cache)
-    # Force English transcription – never translate
-    forced = processor.get_decoder_prompt_ids(language="english", task="transcribe")
-    model.config.forced_decoder_ids = forced
-    model.to("cpu")
-    model.eval()
-    logger.info("[STARTUP] Whisper loaded ✔")
+##############################################################################
 
-
-############################################
-# ── Helpers ───────────────────────────────
-############################################
+# Global ASR (lazy-loaded)
+processor = model = None
 
 def build_prompt(question: str) -> str:
-    """Craft a prompt that elicits concise, structured answers."""
     return (
-        "You are a helpful career‑coach AI. Answer the following interview "
-        "question clearly and concisely, offering practical insights when appropriate.\n"
-        "Use markdown for **bold**, *italic*, and bullet‑lists when helpful. \n"
-        "Ensure your answer is less than 200 words.\n\n"
-        f"Interview question: \"{question}\""
+        "You are a helpful career-coach AI. Answer the following interview "
+        "question clearly and concisely (≤200 words). Use markdown when helpful.\n\n"
+        f"Interview question: \"{question.strip()}\""
     )
 
-def memory_usage_mb() -> float:
-    return psutil.Process().memory_info().rss / 1_048_576  # bytes→MiB
+def memory_mb() -> float:
+    return round(psutil.Process().memory_info().rss / 1_048_576, 1)
 
-############################################
-# ── Routes ────────────────────────────────
-############################################
+@app.on_event("startup")
+async def load_models():
+    global processor, model
+    cache = Path("model_cache"); cache.mkdir(exist_ok=True)
+    processor = WhisperProcessor.from_pretrained(ASR_MODEL_ID, cache_dir=cache)
+    model     = WhisperForConditionalGeneration.from_pretrained(ASR_MODEL_ID, cache_dir=cache)
+    forced    = processor.get_decoder_prompt_ids(task="transcribe", language="english")
+    model.config.forced_decoder_ids = forced
+    model.to("cpu").eval()
+    log.info("🔊 Whisper loaded ✔")
 
 @app.get("/")
-async def root() -> FileResponse:
-    """Serve the single‑page app."""
+async def root() -> FileResponse: # serve SPA
     logger.info("[STATIC] Serving frontend")
     return FileResponse(Path("statics/index.html"))
 
+##############################################################################
+# ── MAIN ENDPOINTS ──────────────────────────────────────────────────────────
+
+def call_gemini(prompt: str, vision_parts=None) -> str:
+    client = genai.Client(api_key=GEMINI_API_KEY)
+    kwargs: Dict = {}
+    if vision_parts:  # multimodal call
+        kwargs["contents"] = vision_parts + [{"text": prompt}]
+    else:
+        kwargs["contents"] = prompt
+    resp = client.models.generate_content(
+        model="gemini-2.5-flash-preview-04-17", **kwargs
+    )
+    return resp.text.strip()
 
 @app.post("/voice-transcribe")
-async def voice_transcribe(file: UploadFile = File(...)):  # noqa: B008
-    """Receive audio, transcribe, push to Gemini, return answer."""
+async def voice_transcribe(file: UploadFile = File(...)):
     if file.content_type not in {"audio/wav", "audio/x-wav", "audio/mpeg"}:
-        raise HTTPException(status_code=415, detail="Unsupported audio type")
-    # Save to a temp file (Whisper expects a filename/bytes)
+        raise HTTPException(415, "Unsupported audio type")
+    # Write temporary audio file
     with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp:
-        tmp.write(await file.read())
-        tmp_path = tmp.name
+        tmp.write(await file.read()); tmp_path = tmp.name
+    # Audio processing and transcription
     try:
-        # ── 1. Transcribe
-        seg  = AudioSegment.from_file(tmp_path).set_frame_rate(SAMPLE_RATE).set_channels(1)
-        audio = np.array(seg.get_array_of_samples()).astype(np.float32) / (2**15)
+        seg = AudioSegment.from_file(tmp_path).set_frame_rate(SAMPLE_RATE).set_channels(1)
+        audio = np.array(seg.get_array_of_samples()).astype(np.float32) / (2 ** 15)
         inputs = processor(audio, sampling_rate=SAMPLE_RATE, return_tensors="pt")
         ids    = model.generate(inputs.input_features.to(model.device))
         question = processor.decode(ids[0], skip_special_tokens=True).strip()
-        if not question: raise ValueError("Could not detect speech")
-        logger.info(f"[VOICE] Detected transcribe: {question}")
-        # ── 2. LLM answer
-        prompt = build_prompt(question)
-        # Gemini Flash 2.5 – tuned for short latency
-        client = genai.Client(api_key=GEMINI_API_KEY)
-        response = client.models.generate_content(
-                model="gemini-2.5-flash-preview-04-17",
-                contents=prompt
-            )
-        answer = response.text.strip()
-        logger.info(f"[LLM] Decision answer: {answer}")
-        return JSONResponse(
-            {
-                "question": question,
-                "answer": answer,
-                "memory_mb": round(memory_usage_mb(), 1),
-            }
-        )
+        if not question:
+            raise ValueError("No speech detected")
+
+        answer = call_gemini(build_prompt(question))
+        return JSONResponse({"question": question, "answer": answer, "memory_mb": memory_mb()})
     finally:
-        os.remove(tmp_path) # Rm audio when done
+        os.remove(tmp_path)
+
+@app.post("/image-question")
+async def image_question(file: UploadFile = File(...)):
+    if file.content_type not in {"image/png", "image/jpeg"}:
+        raise HTTPException(415, "Unsupported image type")
+    # Read file and decode
+    raw = await file.read()
+    b64 = base64.b64encode(raw).decode()
+    # Send image data
+    vision_part = [{
+        "inline_data": {
+            "mime_type": file.content_type,
+            "data": b64
+        }
+    }]
+    # Ask Gemini to return JSON so we can split Q & A
+    prompt = (
+        "From the screenshot extract the interview question (English). "
+        "Then answer concisely (≤200 words). "
+        "Return pure JSON: {\"question\":\"...\",\"answer\":\"...\"}"
+    )
+    # Send prompt and image
+    text = call_gemini(prompt, vision_part)
+    try:
+        parsed = json.loads(text)
+        question, answer = parsed["question"], parsed["answer"]
+    except (json.JSONDecodeError, KeyError):
+        # Fallback: treat whole reply as answer
+        question, answer = "[Extracted from screenshot]", text
+    return JSONResponse({"question": question, "answer": answer, "memory_mb": memory_mb()})
+
+@app.post("/text-question")
+async def text_question(payload: Dict):
+    question = (payload.get("question") or "").strip()
+    if not question:
+        raise HTTPException(400, "question is required")
+    answer = call_gemini(build_prompt(question))
+    return JSONResponse({"question": question, "answer": answer, "memory_mb": memory_mb()})

requirements.txt

@@ -1,23 +1,24 @@
 # Core server
 fastapi
 uvicorn[standard]
-aiofiles               # Static file serving
-python-multipart       # File uploads
+aiofiles               # Static files
 
 # Voice‑to‑text (Whisper via Transformers)
-transformers            # For whisper
-torch                   # Just to run transformer so don't remove
-huggingface_hub
+transformers           # For whisper
+torch                  # Just to run transformer so don't remove
 accelerate
 
-# Audio
+# Audio & Image
 pydub
 ffmpeg-python
-openai-whisper          # pulls tiny‑en / small‑en
+openai-whisper         # pulls tiny‑en / small‑en
+pillow
 
 # Gemini Flash 2.5
 google-genai
-python-dotenv          # Optional – read GOOGLE_API_KEY
 
 # Utilities
-psutil                 # Lightweight health logging
+psutil                 # Lightweight health logging
+python-multipart       # File uploads
+huggingface_hub
+python-dotenv

statics/index.html

@@ -13,11 +13,16 @@
 <body>
   <main class="container">
     <h1>Interview Q&amp;A Assistant</h1>
-    <p class="subtitle">Hold the button, ask your interview question, release to get an answer.</p>
-    <button id="record-button" class="record-btn">🎙 Hold&nbsp;to&nbsp;Ask</button>
-    <section class="output-section">
-      <h2>Your Question</h2>
-      <pre id="question-output" class="output"></pre>
+    <p class="subtitle">🎙&nbsp;Hold the button, ask your interview question, release to get an answer.<br>
+      📸&nbsp;Or upload a screenshot of the question.</p>
+      <button id="record-button"     class="record-btn">🎤 Hold&nbsp;to&nbsp;Ask</button>
+      <button id="screenshot-button" class="screenshot-btn">📸 Drop&nbsp;your&nbsp;Screenshot</button>
+      <input  id="file-input" type="file" accept="image/*" hidden />
+      <section class="output-section">
+      <h2>Your Question
+        <img src="/statics/edit.png" id="edit-btn" class="edit-icon" title="Edit & re-ask">
+      </h2>
+      <div id="question-output" class="output" contenteditable="false"></div>
       <h2>AI&nbsp;Answer</h2>
       <pre id="answer-output" class="output markdown"></pre>
     </section>

statics/script.js

@@ -1,13 +1,14 @@
 /*******************************
  * Interview Q&A Frontend JS   *
  *******************************/
+const recordBtn      = document.getElementById("record-button");
+const screenshotBtn  = document.getElementById("screenshot-button");
+const fileInput      = document.getElementById("file-input");
+const questionEl     = document.getElementById("question-output");
+const answerEl       = document.getElementById("answer-output");
+const editBtn        = document.getElementById("edit-btn");
 
-// Elements
-const recordBtn   = document.getElementById("record-button");
-const questionEl  = document.getElementById("question-output");
-const answerEl    = document.getElementById("answer-output");
-
-// Typing animation util
+/* ─────────────────── Typing effect utility ─────────────────── */
 function typeEffect(el, text, speed = 30) {
   el.textContent = "";
   let idx = 0;
@@ -18,53 +19,121 @@ function typeEffect(el, text, speed = 30) {
   }, speed);
 }
 
-// Audio recording setup
+/* ─────────────────── Abort-controller wrapper ───────────────── */
+let currentController = null;
+function fetchWithAbort(url, opts = {}) {
+  if (currentController) currentController.abort();      // cancel previous req
+  currentController = new AbortController();
+  return fetch(url, { ...opts, signal: currentController.signal });
+}
+
+/* ─────────────────── Audio recording setup ─────────────────── */
 let mediaRecorder, chunks = [];
-// Initialise media data
 async function initMedia() {
+  const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+  mediaRecorder = new MediaRecorder(stream);
+
+  mediaRecorder.ondataavailable = e => chunks.push(e.data);
+
+  mediaRecorder.onstop = async () => {
+    const audioBlob = new Blob(chunks, { type: "audio/wav" });
+    chunks = [];
+
+    const form = new FormData();
+    form.append("file", audioBlob, "record.wav");
+
+    questionEl.textContent = "⌛ Transcribing…";
+    answerEl.innerHTML = "";
+
+    try {
+      const res  = await fetchWithAbort("/voice-transcribe", { method: "POST", body: form });
+      if (!res.ok) throw new Error(`HTTP ${res.status}`);
+      const data = await res.json();
+      displayQa(data);
+    } catch (err) {
+      answerEl.textContent = "❌ " + err.message;
+    }
+  };
+}
+
+/* ─────────────── Screenshot / image-question upload ─────────── */
+fileInput.addEventListener("change", async (e) => {
+  const file = e.target.files[0];
+  if (!file) return;
+  const form = new FormData();
+  form.append("file", file);
+
+  questionEl.textContent = "⌛ Processing screenshot…";
+  answerEl.innerHTML = "";
+
   try {
-    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
-    mediaRecorder = new MediaRecorder(stream);
-    // Continuously push available data
-    mediaRecorder.ondataavailable = e => chunks.push(e.data);
-    // Stop btn start write and send audio chunk
-    mediaRecorder.onstop = async () => {
-      const audioBlob = new Blob(chunks, { type: "audio/wav" });
-      chunks = [];
-      // Build form data
-      const form = new FormData();
-      form.append("file", audioBlob, "record.wav");
-      // UX feedback
-      typeEffect(questionEl, "⌛ Transcribing…");
-      answerEl.textContent = "";
-      try {
-        const res = await fetch("/voice-transcribe", { method: "POST", body: form });
-        if (!res.ok) throw new Error(`HTTP ${res.status}`);
-        const data = await res.json();
-        // render markdown after a small delay for dramatic effect
-        typeEffect(questionEl, data.question || "[no speech detected]");
-        setTimeout(() => typeEffect(answerEl, data.answer || "[no answer]"), 500);
-      } catch (err) {
-        typeEffect(answerEl, "❌ " + err.message);
-      }
-    };
+    const res  = await fetchWithAbort("/image-question", { method: "POST", body: form });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    const data = await res.json();
+    displayQa(data);
   } catch (err) {
-    alert("Microphone access denied – please allow permissions.");
+    answerEl.textContent = "❌ " + err.message;
+  } finally {
+    fileInput.value = ""; // reset for next upload
   }
-}
+});
+screenshotBtn.addEventListener("click", () => fileInput.click());
 
-// Hold‑to‑record UX
+/* ─────────────────── Hold-to-record UX ─────────────────────── */
 function bindRecordBtn() {
-  if (!mediaRecorder) return;
   recordBtn.addEventListener("mousedown", () => mediaRecorder.start());
   recordBtn.addEventListener("mouseup",   () => mediaRecorder.stop());
-  // Touch devices
   recordBtn.addEventListener("touchstart", e => { e.preventDefault(); mediaRecorder.start(); });
-  recordBtn.addEventListener("touchend",   e => { e.preventDefault(); mediaRecorder.stop(); });
+  recordBtn.addEventListener("touchend",   e => { e.preventDefault(); mediaRecorder.stop();  });
 }
 
-// Init on page load
+/* ─────────────────── Editable question block ───────────────── */
+function enableEdit() {
+  questionEl.contentEditable = "true";
+  questionEl.classList.add("editing");
+  questionEl.focus();
+}
+
+async function sendEditedQuestion(text) {
+  questionEl.contentEditable = "false";
+  questionEl.classList.remove("editing");
+  answerEl.textContent = "⌛ Thinking…";
+  try {
+    const res = await fetchWithAbort("/text-question", {
+      method : "POST",
+      headers: { "Content-Type": "application/json" },
+      body   : JSON.stringify({ question: text })
+    });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    const data = await res.json();
+    displayQa(data);
+  } catch (err) {
+    answerEl.textContent = "❌ " + err.message;
+  }
+}
+
+editBtn.addEventListener("click", () => enableEdit());
+questionEl.addEventListener("keydown", (e) => {
+  if (e.key === "Enter") {
+    e.preventDefault();
+    const text = questionEl.innerText.trim();
+    if (text) sendEditedQuestion(text);
+  }
+});
+
+/* ─────────────────────── helpers ───────────────────────────── */
+function displayQa(data) {
+  typeEffect(questionEl, data.question || "[no question]");
+  const html = DOMPurify.sanitize(marked.parse(data.answer || ""));
+  setTimeout(() => { answerEl.innerHTML = html; }, 400);  // nice stagger
+}
+
+/* ─────────────────────── bootstrap ─────────────────────────── */
 window.addEventListener("DOMContentLoaded", async () => {
-  try { await initMedia(); bindRecordBtn(); }
-  catch (e) { alert("Mic permission required"); }
-});
+  try {
+    await initMedia();
+    bindRecordBtn();
+  } catch {
+    alert("Microphone permission is required.");
+  }
+});

statics/styles.css

@@ -1,41 +1,34 @@
-/* Simple, clean aesthetic */
 :root {
-  --primary: #0052cc;
-  --accent:  #ff6666;
-  --bg:      #f8f9fc;
-  --mono:    "Courier New", monospace;
+  --primary:#0052cc; --accent:#ff4d4d; --sub:#66adff; --bg:#f8f9fc;
+  --mono:"Courier New",monospace;
 }
 
-html,body {
-  margin: 0; padding: 0; background: var(--bg); font-family: Arial, sans-serif;
-}
-
-.container {
-  max-width: 720px; margin: 40px auto; padding: 24px;
-  background: #fff; border-radius: 8px; box-shadow: 0 4px 12px rgba(0,0,0,.08);
-}
-
-h1 { margin-top: 0; text-align: center; color: var(--primary); }
-.subtitle { text-align: center; color: #444; margin-bottom: 32px; }
-
-.record-btn {
-  display: block; margin: 0 auto 24px; padding: 14px 28px;
-  background: var(--accent); color: #fff; border: none; border-radius: 50px;
-  font-size: 17px; cursor: pointer; transition: background .25s;
-}
-
-.record-btn:hover { background: #ff4d4d; }
-
-.output-section h2 { margin: 24px 0 8px; color: var(--primary); }
-
-.output {
-  background: #000; color: #0f0; padding: 16px; min-height: 60px;
-  border-radius: 4px; overflow-x: auto; font-family: var(--mono);
-  white-space: pre-wrap; word-wrap: break-word;
-}
-
-/* --- new markdown styling --- */
-.output.markdown h3, .output.markdown h2 { color:#55f; margin:6px 0; }
-.output.markdown strong   { font-weight:bold; color:#fff; background:#333; padding:0 4px; border-radius:3px; }
-.output.markdown em       { font-style:italic;  color:#ffd700; }
-.output.markdown ul       { margin:4px 0 4px 20px; }
+/* Layout -----------------------------------------------------------------*/
+html,body{margin:0;padding:0;background:var(--bg);font-family:Arial,Helvetica,sans-serif}
+.container{max-width:720px;margin:20px auto;padding:24px;background:#fff;
+  border-radius:8px;box-shadow:0 4px 12px rgba(0,0,0,.08)}
+h1{margin-top:0;text-align:center;color:var(--primary)}
+.subtitle{text-align:center;color:#444;margin-bottom:16px}
+
+/* Buttons ----------------------------------------------------------------*/
+.record-btn,.screenshot-btn{
+  display:block;margin:0 auto 10px;padding:14px 28px;border:none;border-radius:50px;
+  font-size:17px;color:#fff;cursor:pointer;transition:background .25s}
+.record-btn   {background:var(--accent)}   .record-btn:hover   {background:#b80e0e}
+.screenshot-btn{background:var(--sub)}     .screenshot-btn:hover{background:#1f5089}
+
+/* Output blocks -----------------------------------------------------------*/
+.output-section h2{margin:22px 0 8px;color:var(--primary);display:flex;align-items:center;gap:6px}
+.output{
+  background:#000;color:#0f0;padding:16px;min-height:60px;border-radius:4px;
+  overflow-x:auto;font-family:var(--mono);white-space:pre-wrap;word-wrap:break-word}
+
+/* Edit icon ---------------------------------------------------------------*/
+.edit-icon{width:18px;height:18px;cursor:pointer;opacity:.6;transition:opacity .2s}
+.edit-icon:hover{opacity:1}
+
+/* Markdown tweaks ---------------------------------------------------------*/
+.output.markdown h3,.output.markdown h2{color:#55f;margin:6px 0}
+.output.markdown strong{font-weight:bold;color:#fff;background:#333;padding:0 4px;border-radius:3px}
+.output.markdown em{font-style:italic;color:#ffd700}
+.output.markdown ul{margin:4px 0 4px 20px}