Spaces:
Runtime error
Runtime error
File size: 1,556 Bytes
3c5f44b 9011670 3c5f44b 6213ab1 3c5f44b a1523c0 3c5f44b 9d7659b a1523c0 3c5f44b |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 |
from langchain.vectorstores import Qdrant
from langchain_huggingface import HuggingFaceEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.schema import Document
import os
import fitz # PyMuPDF
from config import EMBEDDING_MODEL,QDRANT_HOST,QDRANT_API_KEY
embedding_model = HuggingFaceEmbeddings(model_name=EMBEDDING_MODEL)
def extract_text_from_pdf(pdf_path):
if not os.path.exists(pdf_path):
raise FileNotFoundError(f"File not found: {pdf_path}")
doc = fitz.open(pdf_path)
text = "\n".join([page.get_text("text") for page in doc])
return text
def load_pdf_data(pdf_path):
text = extract_text_from_pdf(pdf_path)
splitter = RecursiveCharacterTextSplitter(
chunk_size=2000,
chunk_overlap=100,
)
chunks = splitter.split_text(text)
documents = [
Document(page_content=chunk, metadata={"source": pdf_path})
for chunk in chunks
]
return documents
def get_vector_db():
qdrant_url = QDRANT_HOST
api_key = QDRANT_API_KEY
collection_name = "discvr_embeddings"
docs = load_pdf_data("data/Explorer.pdf")
vector_db = Qdrant.from_documents(
docs, embedding_model,
location=qdrant_url,
collection_name=collection_name,
api_key=api_key,
timeout=500
)
return vector_db
def retrieve_info(query, k=20):
vector_db = get_vector_db()
docs = vector_db.similarity_search(query, k=k)
return "\n".join([doc.page_content for doc in docs]) |