Update scraper.py

scraper.py

@@ -6,14 +6,23 @@ import logging
 import os
 import time
 import random
+import pandas as pd
 from config import SCRAPER_TIMEOUT, CHROME_DRIVER_PATH, SCRAPER_MAX_RETRIES
 
 
+USER_AGENTS = us_ag = pd.read_csv("https://gist.githubusercontent.com/pzb/b4b6f57144aea7827ae4/raw/cf847b76a142955b1410c8bcef3aabe221a63db1/user-agents.txt", sep="\t", header=None)
+USER_AGENTS = USER_AGENTS.iloc[:, 0].copy()
+
+
 def get_text(url, n_words=15):
     try:
         driver = None
         logging.warning(f"Initiated Scraping {url}")
-        user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
+        # user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36"
+        ua = USER_AGENTS[np.random.randint(low=0, high=len(USER_AGENTS), size=1)]
+        ua = ua.reset_index(drop=True)
+        ua = ua[0]
+        user_agent = ua
         options = uc.ChromeOptions() 
         options.add_argument("--headless") 
         options.add_argument(f"user-agent={user_agent}")