fix: Update dependencies and imports

- Update LangChain imports to use community packages
- Fix sherlock package dependency
- Simplify OSINT engine implementation

engines/osint.py

@@ -1,167 +1,151 @@
 """
-OSINT engine for comprehensive information gathering.
+OSINT engine for gathering intelligence from various sources.
 """
 from typing import Dict, List, Any, Optional
 import asyncio
 import json
-from dataclasses import dataclass
-import holehe.core as holehe
-from sherlock import sherlock
-import face_recognition
-import numpy as np
-from PIL import Image
-import io
-import requests
+from datetime import datetime
+import whois
+from holehe.core import import_submodules
+from holehe.core import get_functions
 from geopy.geocoders import Nominatim
 from geopy.exc import GeocoderTimedOut
-import whois
-from datetime import datetime
+import python_sherlock
+from python_sherlock import sherlock_module
 from tenacity import retry, stop_after_attempt, wait_exponential
 
-@dataclass
-class PersonInfo:
-    name: str
-    age: Optional[int] = None
-    location: Optional[str] = None
-    gender: Optional[str] = None
-    social_profiles: List[Dict[str, str]] = None
-    images: List[str] = None
-    
-    def to_dict(self) -> Dict[str, Any]:
-        return {
-            "name": self.name,
-            "age": self.age,
-            "location": self.location,
-            "gender": self.gender,
-            "social_profiles": self.social_profiles or [],
-            "images": self.images or []
-        }
-
 class OSINTEngine:
     def __init__(self):
-        self.geolocator = Nominatim(user_agent="intelligent_search_engine")
-        self.known_platforms = [
-            "Twitter", "Instagram", "Facebook", "LinkedIn", "GitHub",
-            "Reddit", "YouTube", "TikTok", "Pinterest", "Snapchat",
-            "Twitch", "Medium", "Dev.to", "Stack Overflow"
-        ]
-    
-    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
+        self.holehe_modules = import_submodules("holehe.modules")
+        self.holehe_functions = get_functions(self.holehe_modules)
+        self.geolocator = Nominatim(user_agent="my_osint_app")
+        
     async def search_username(self, username: str) -> Dict[str, Any]:
         """Search for username across multiple platforms."""
-        results = []
-        
-        # Use holehe for email-based search
-        email = f"{username}@gmail.com"  # Example email
-        holehe_results = await holehe.check_email(email)
-        
-        # Use sherlock for username search
-        sherlock_results = sherlock.sherlock(username, self.known_platforms, verbose=False)
-        
-        # Combine results
-        for platform, data in {**holehe_results, **sherlock_results}.items():
-            if data.get("exists", False):
-                results.append({
-                    "platform": platform,
-                    "url": data.get("url", ""),
-                    "confidence": data.get("confidence", "high")
-                })
-        
-        return {
-            "username": username,
-            "found_on": results
+        results = {
+            "found": [],
+            "not_found": [],
+            "errors": []
         }
-    
-    async def search_person(self, name: str, location: Optional[str] = None, 
-                          age: Optional[int] = None, gender: Optional[str] = None) -> PersonInfo:
-        """Search for information about a person."""
-        person = PersonInfo(
-            name=name,
-            age=age,
-            location=location,
-            gender=gender
-        )
         
-        # Initialize social profiles list
-        person.social_profiles = []
-        
-        # Search for social media profiles
-        username_variants = [
-            name.replace(" ", ""),
-            name.replace(" ", "_"),
-            name.replace(" ", "."),
-            name.lower().replace(" ", "")
-        ]
+        # Sherlock search
+        try:
+            sherlock_results = sherlock_module.search_username(username)
+            for site, data in sherlock_results.items():
+                if data.get("status") == "found":
+                    results["found"].append({
+                        "platform": site,
+                        "url": data.get("url", ""),
+                        "source": "sherlock"
+                    })
+                elif data.get("status") == "not found":
+                    results["not_found"].append(site)
+                else:
+                    results["errors"].append(site)
+        except Exception as e:
+            print(f"Sherlock error: {e}")
         
-        for username in username_variants:
-            results = await self.search_username(username)
-            person.social_profiles.extend(results.get("found_on", []))
+        # Holehe search
+        try:
+            holehe_tasks = []
+            for platform in self.holehe_functions:
+                holehe_tasks.append(platform(username))
+            
+            holehe_results = await asyncio.gather(*holehe_tasks, return_exceptions=True)
+            
+            for result in holehe_results:
+                if isinstance(result, Exception):
+                    continue
+                    
+                if result.get("exists"):
+                    results["found"].append({
+                        "platform": result.get("name", "unknown"),
+                        "url": result.get("url", ""),
+                        "source": "holehe"
+                    })
+                else:
+                    results["not_found"].append(result.get("name", "unknown"))
+        except Exception as e:
+            print(f"Holehe error: {e}")
         
-        return person
+        return results
     
     @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
-    async def analyze_image(self, image_data: bytes) -> Dict[str, Any]:
-        """Analyze an image for faces and other identifiable information."""
+    async def search_domain(self, domain: str) -> Dict[str, Any]:
+        """Get information about a domain."""
         try:
-            # Load image
-            image = face_recognition.load_image_file(io.BytesIO(image_data))
-            
-            # Detect faces
-            face_locations = face_recognition.face_locations(image)
-            face_encodings = face_recognition.face_encodings(image, face_locations)
-            
-            results = {
-                "faces_found": len(face_locations),
-                "faces": []
+            w = whois.whois(domain)
+            return {
+                "registrar": w.registrar,
+                "creation_date": w.creation_date,
+                "expiration_date": w.expiration_date,
+                "last_updated": w.updated_date,
+                "status": w.status,
+                "name_servers": w.name_servers,
+                "emails": w.emails
             }
-            
-            # Analyze each face
-            for i, (face_encoding, face_location) in enumerate(zip(face_encodings, face_locations)):
-                face_data = {
-                    "location": {
-                        "top": face_location[0],
-                        "right": face_location[1],
-                        "bottom": face_location[2],
-                        "left": face_location[3]
-                    }
-                }
-                results["faces"].append(face_data)
-            
-            return results
         except Exception as e:
             return {"error": str(e)}
     
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
     async def search_location(self, location: str) -> Dict[str, Any]:
-        """Gather information about a location."""
+        """Get information about a location."""
         try:
-            # Geocode the location
             location_data = self.geolocator.geocode(location, timeout=10)
-            
-            if not location_data:
-                return {"error": "Location not found"}
-            
-            return {
-                "address": location_data.address,
-                "latitude": location_data.latitude,
-                "longitude": location_data.longitude,
-                "raw": location_data.raw
-            }
+            if location_data:
+                return {
+                    "address": location_data.address,
+                    "latitude": location_data.latitude,
+                    "longitude": location_data.longitude,
+                    "raw": location_data.raw
+                }
+            return {"error": "Location not found"}
         except GeocoderTimedOut:
             return {"error": "Geocoding service timed out"}
         except Exception as e:
             return {"error": str(e)}
     
-    async def analyze_domain(self, domain: str) -> Dict[str, Any]:
-        """Analyze a domain for WHOIS and other information."""
+    async def search_person(self, name: str, location: Optional[str] = None) -> Dict[str, Any]:
+        """Search for information about a person."""
+        results = {
+            "name": name,
+            "location": location,
+            "social_profiles": [],
+            "possible_emails": [],
+            "location_info": None
+        }
+        
+        # Get location information if provided
+        if location:
+            results["location_info"] = await self.search_location(location)
+        
+        # Generate possible email formats
+        name_parts = name.lower().split()
+        if len(name_parts) >= 2:
+            first, last = name_parts[0], name_parts[-1]
+            common_domains = ["gmail.com", "yahoo.com", "hotmail.com", "outlook.com"]
+            email_formats = [
+                f"{first}.{last}@{domain}",
+                f"{first}{last}@{domain}",
+                f"{first[0]}{last}@{domain}",
+                f"{first}_{last}@{domain}"
+            ]
+            results["possible_emails"] = email_formats
+        
+        return results
+    
+    async def search(self, query: str, search_type: str = "username") -> Dict[str, Any]:
+        """Main search interface."""
         try:
-            w = whois.whois(domain)
-            return {
-                "registrar": w.registrar,
-                "creation_date": w.creation_date,
-                "expiration_date": w.expiration_date,
-                "last_updated": w.updated_date,
-                "status": w.status,
-                "name_servers": w.name_servers
-            }
+            if search_type == "username":
+                return await self.search_username(query)
+            elif search_type == "domain":
+                return await self.search_domain(query)
+            elif search_type == "location":
+                return await self.search_location(query)
+            elif search_type == "person":
+                return await self.search_person(query)
+            else:
+                return {"error": f"Unknown search type: {search_type}"}
         except Exception as e:
             return {"error": str(e)}

engines/search.py

@@ -3,9 +3,8 @@ Advanced RAG-based search engine with multi-source intelligence.
 """
 from typing import List, Dict, Any, Optional
 import asyncio
-from langchain.chains import RetrievalQAWithSourcesChain
-from langchain.embeddings import HuggingFaceEmbeddings
-from langchain.vectorstores import FAISS
+from langchain_community.embeddings import HuggingFaceEmbeddings
+from langchain_community.vectorstores import FAISS
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.docstore.document import Document
 from duckduckgo_search import DDGS

requirements.txt

@@ -1,8 +1,12 @@
 # Core Dependencies
 python-dotenv>=1.0.0
-langchain>=0.0.200
+langchain>=0.1.0
+langchain-community>=0.0.10
+transformers>=4.30.2
 sentence-transformers>=2.2.2
 faiss-cpu>=1.7.4
+torch>=2.0.1 --index-url https://download.pytorch.org/whl/cpu
+accelerate>=0.21.0
 
 # Web Scraping & Search
 duckduckgo-search>=3.8.3
@@ -13,8 +17,15 @@ tenacity>=8.2.2
 aiohttp>=3.8.5
 httpx>=0.24.1
 
+# Image Processing
+Pillow>=10.0.0
+face-recognition>=1.3.0
+opencv-python-headless>=4.8.0
+
 # OSINT Tools
 holehe>=1.61
+sherlock-project>=0.14.0
+python-sherlock>=0.1.0
 python-whois>=0.8.0
 geopy>=2.3.0