Update app.py

app.py

@@ -7,6 +7,7 @@ from markitdown import MarkItDown
 import tempfile
 import os
 import asyncio
+import re
 
 app = FastAPI()
 
@@ -19,13 +20,13 @@ async def stealthy_scraper(url):
     """Fetches HTML content using StealthyFetcher (already async internally)."""
     # Use await if async_fetch is available, otherwise keep .fetch
     html = await stealthy_fetcher.async_fetch(url)  #Corrected to async_fetch
-    return html.html_content
+    return html
 
 
 async def scraper(url):
     """Fetches HTML content using AsyncFetcher."""
     html = await async_fetcher.get(url)  # Use await for async operations
-    return html.html_content
+    return html
 
 
 async def convert_html_to_md(html):
@@ -42,6 +43,12 @@ async def convert_html_to_md(html):
     await asyncio.to_thread(os.remove, temp_file_path)
     return x
 
+async def convert_html_to_text(html):
+    """Converts HTML to Text using re."""
+    html = re.sub(r'\n+', ' ', html) 
+    html = re.sub(r'\s+', ' ', html)
+    return html
+
 
 @app.get("/read/{url:path}", response_class=PlainTextResponse)
 async def get_markdown_get(request: Request, url: str):
@@ -53,7 +60,22 @@ async def get_markdown_get(request: Request, url: str):
         if not full_url.startswith(('http://', 'https://')):
             full_url = f"http://{full_url}"
 
-        markdown_output = await convert_html_to_md(await scraper(full_url))
+        markdown_output = await convert_html_to_md(await scraper(full_url).html_content)
+        return PlainTextResponse(markdown_output)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error processing URL: {e}")
+
+@app.get("/read/text/{url:path}", response_class=PlainTextResponse)
+async def get_markdown_get(request: Request, url: str):
+    """Handles GET requests to /read/{url}, returning Markdown content."""
+    try:
+        full_url = str(request.url)
+        full_url = full_url.split("/read/")[1]
+
+        if not full_url.startswith(('http://', 'https://')):
+            full_url = f"http://{full_url}"
+
+        markdown_output = await convert_html_to_md(await scraper(full_url).get_all_text())
         return PlainTextResponse(markdown_output)
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error processing URL: {e}")
@@ -69,7 +91,7 @@ async def get_markdown_get_stealthy(request: Request, url: str):  # Renamed for
         if not full_url.startswith(('http://', 'https://')):
             full_url = f"http://{full_url}"
 
-        markdown_output = await convert_html_to_md(await stealthy_scraper(full_url))
+        markdown_output = await convert_html_to_md(await stealthy_scraper(full_url).html_content)
         return PlainTextResponse(markdown_output)
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error processing URL: {e}")