added a scrape route

app.py

@@ -16,6 +16,7 @@ from pathlib import Path
 from collections import Counter, defaultdict
 from utils.logger import log_request
 from chipsearch.main import search
+from scrape.main import scrape_to_markdown
 
 app = FastAPI()
 
@@ -424,3 +425,16 @@ async def chipsearch(request: Request):
         unique=bool(request.query_params.get("unique", False))
     )
     return data
+
+
+@app.post("/scrape-md")
+async def scrape_md(request: Request):
+    data = await request.json()
+    url = data.get("url")
+    if not url:
+        return {"error": "URL is required"}
+    
+    data = scrape_to_markdown(url)
+
+    return {"markdown": data}
+

logs.json

@@ -1,317 +0,0 @@
-[
-  {
-    "timestamp": "2025-04-12T11:48:38.183002",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T11:49:55.976382",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T11:51:54.228166",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T11:59:54.231283",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:00:40.844595",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:00:45.093082",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:00:53.603673",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:01:08.563064",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:01:11.525946",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:01:14.278740",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:01:16.915702",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:01:30.033848",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:02:15.117096",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:02:23.649030",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:02:28.933558",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:02:38.160605",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:02:52.076861",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:04:02.320142",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:04:05.385644",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:04:20.976615",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:04:24.590270",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:04:34.063593",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:04:38.292885",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:04:40.796568",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:05:13.292850",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:05:23.222112",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:05:28.215528",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:05:32.617464",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:06:42.995442",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:06:44.829603",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:06:52.601184",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:06:56.874556",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:12:39.624859",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:12:44.753091",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:12:49.680739",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:12:57.229754",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:00.767253",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:03.882938",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:07.323707",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:15.612926",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:17.961841",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:22.519624",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:31.175155",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:38.396994",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:42.230876",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:45.023945",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:48.831325",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:51.526062",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:13:56.833256",
-    "endpoint": "/chat",
-    "query": "groqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:14:02.725289",
-    "endpoint": "/chat",
-    "query": "vercelXaigenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:14:12.979970",
-    "endpoint": "/chat",
-    "query": "generate"
-  },
-  {
-    "timestamp": "2025-04-12T12:14:34.965307",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:14:41.576413",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:15:29.506084",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:17:00.139827",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:18:07.735115",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:19:21.687931",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:22:30.826896",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:22:33.378368",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:22:35.429037",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:22:37.216791",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:23:02.229775",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  },
-  {
-    "timestamp": "2025-04-12T12:23:41.708982",
-    "endpoint": "/chat",
-    "query": "vercelGroqgenerate"
-  }
-]

requirements.txt

@@ -9,4 +9,5 @@ jinja2
 aiofiles
 matplotlib
 curl_cffi
-beautifulsoup4
+beautifulsoup4
+html2text

scrape/main.py

@@ -0,0 +1,29 @@
+from curl_cffi import requests as req
+from bs4 import BeautifulSoup
+import html2text
+
+def scrape_to_markdown(url):
+    """
+    Scrapes a webpage and converts its content to markdown format.
+    
+    Args:
+        url (str): The URL of the webpage to scrape
+        
+    Returns:
+        str: The webpage content converted to markdown
+    """
+    # Fetch HTML content
+    response = req.get(url, impersonate='chrome110')
+    soup = BeautifulSoup(response.text, 'html.parser')
+    
+    # Clean up unwanted tags
+    for tag in soup(['script', 'style', 'noscript', 'svg', 'css']):
+        tag.decompose()
+    
+    # Extract cleaned HTML
+    clean_html = str(soup)
+    
+    # Convert to Markdown
+    markdown = html2text.html2text(clean_html)
+    
+    return markdown

test.py

@@ -1,50 +1,22 @@
-import requests
-import json
+from curl_cffi import requests as req
+from bs4 import BeautifulSoup
+import html2text
 
-# import requests
-# import json
+url = 'https://www.firecrawl.dev/'
 
-# url = "http://127.0.0.1:8000/chat"
+# Fetch HTML content
+response = req.get(url)
+soup = BeautifulSoup(response.text, 'html.parser')
 
-# payload = {
-#     "model": "your-model-name",
-#     "message": "Create a json object of top 10 anime! answer in json only!",
-#     "messages": []
-# }
+# Optional: Clean up unwanted tags
+for tag in soup(['script', 'style', 'noscript', 'svg']):
+    tag.decompose()
 
-# headers = {
-#     "Content-Type": "application/json"
-# }
+# Extract cleaned HTML
+clean_html = str(soup)
 
-# # Send streaming POST request
-# with requests.post(url, data=json.dumps(payload), headers=headers, stream=True) as response:
-#     if response.status_code == 200:
-#         for line in response.iter_lines(decode_unicode=True):
-#             if line and line.startswith('data: '):
-#                 try:
-#                     # Remove 'data: ' prefix and parse JSON
-#                     json_data = json.loads(line[6:])
-#                     # Extract text from choices if available
-#                     if json_data.get('choices') and len(json_data['choices']) > 0:
-#                         text = json_data['choices'][0].get('text', '')
-#                         if text:
-#                             print(text, end='')
-#                 except json.JSONDecodeError:
-#                     continue
-#     else:
-#         print("Error:", response.status_code, response.text)
+# Convert to Markdown
+markdown = html2text.html2text(clean_html)
 
-
-def search_chips(term, num_results=10, advanced=False, unique=False):
-    url = f"http://127.0.0.1:8000/chipsearch?term={term}&num_results={num_results}&advanced={advanced}&unique={unique}"
-    
-    try:
-        response = requests.post(url)
-        response.raise_for_status()
-        return response.json()
-    except requests.exceptions.RequestException as e:
-        print(f"Error: {e}")
-        return None
-
-results = search_chips("top 10 anime of all time")
-print(results)
+# Output
+print(markdown)