diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..5a62f7b33683a8e880027fea21fc7042038e2915 --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +# Node artifact files +node_modules/ +dist/ + +# Compiled Java class files +*.class + +# Compiled Python bytecode +*.py[cod] + +# Log files +*.log + +# Package files +*.jar + +# Maven +target/ +dist/ + +# JetBrains IDE +.idea/ + +# Unit test reports +TEST*.xml + +# Generated by MacOS +.DS_Store + +# Generated by Windows +Thumbs.db + +# Applications +*.app +*.exe +*.war + +# Large media files +*.mp4 +*.tiff +*.avi +*.flv +*.mov +*.wmv +# These are some examples of commonly ignored file patterns. +# You should customize this list as applicable to your project. +# Learn more about .gitignore: +# https://www.atlassian.com/git/tutorials/saving-changes/gitignore + +venv/ +.venv/ + +**/__pycache__/ +data/* +env.sh +db.sqlite3 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..8f0dcd6effd6c5fecadb8b72a1879d2068bb7e0e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11 + +WORKDIR /code + +COPY ./requirements.txt /code/requirements.txt + +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt + +COPY . . + +CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] \ No newline at end of file diff --git a/core/__init__.py b/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/core/admin.py b/core/admin.py new file mode 100644 index 0000000000000000000000000000000000000000..b360d150d0ae39271c06861ce9d71b22589dd8fa --- /dev/null +++ b/core/admin.py @@ -0,0 +1,16 @@ +""" +This file is used to register the models in the admin panel. +""" + +from django.contrib import admin +from core.models import MutualFund, Stock + + +@admin.register(MutualFund) +class MutualFundAdmin(admin.ModelAdmin): + list_display = ("id", "rank", "fund_name", "isin_number", "security_id") + + +@admin.register(Stock) +class StockAdmin(admin.ModelAdmin): + pass diff --git a/core/apps.py b/core/apps.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ce093bd64d1542e8df1162e3992e04606716d9 --- /dev/null +++ b/core/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class CoreConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "core" diff --git a/core/clients/__init__.py b/core/clients/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/core/clients/mutual_fund.py b/core/clients/mutual_fund.py new file mode 100644 index 0000000000000000000000000000000000000000..8d34359daf8ff4897fc038c4c44e5931749abb14 --- /dev/null +++ b/core/clients/mutual_fund.py @@ -0,0 +1,267 @@ +""" + +""" +import logging + +from django.conf import settings + +import requests +from bs4 import BeautifulSoup +from core.models import MutualFund +from core.constants import MONEYCONTROL_TOPFUNDS_URL +from data_pipeline.interfaces.api_client import DataClient + + +logger = logging.getLogger(__name__) + + +settings.MORNINGSTAR_API_HEADERS = { + "X-RapidAPI-Key": settings.MORNINGSTAR_KEY, + "X-RapidAPI-Host": settings.MORNINGSTAR_HOST, +} + + +class MFList(DataClient): + model = MutualFund + + # Monring Star List MF url + api_url = "https://lt.morningstar.com/api/rest.svc/g9vi2nsqjb/security/screener?page=1&pageSize=15000&sortOrder=name%20asc&outputType=json&version=1&languageId=en¤cyId=INR&universeIds=FOIND%24%24ALL%7CFCIND%24%24ALL&securityDataPoints=secId%2ClegalName%2CclosePrice%2CclosePriceDate%2Cyield_M12%2CongoingCharge%2CcategoryName%2CMedalist_RatingNumber%2CstarRatingM255%2CreturnD1%2CreturnW1%2CreturnM1%2CreturnM3%2CreturnM6%2CreturnM0%2CreturnM12%2CreturnM36%2CreturnM60%2CreturnM120%2CmaxFrontEndLoad%2CmanagerTenure%2CmaxDeferredLoad%2CexpenseRatio%2Cisin%2CinitialPurchase%2CfundTnav%2CequityStyleBox%2CbondStyleBox%2CaverageMarketCapital%2CaverageCreditQualityCode%2CeffectiveDuration%2CmorningstarRiskM255%2CalphaM36%2CbetaM36%2Cr2M36%2CstandardDeviationM36%2CsharpeM36%2CtrackRecordExtension&filters=&term=" + + def __init__(self) -> None: + self.api_response = None + self.transformed_data = None + + def extract(self): + + super().extract() + + logger.info("Calling Morningstar API") + response = requests.get(self.api_url) + + # Check if the request was successful (status code 200) + if response.status_code == 200: + + # Parse JSON response + self.api_response = response.json() + logger.info( + f'Morningstar API response received {len(self.api_response["rows"])} funds' + ) + + else: + logger.info("Received status code: {response.status_code}") + logger.info(response.json()) + + def transform(self): + """ + Transform the data to the format required by the model + """ + + super().transform() + + self.transformed_data = [ + { + "fund_name": fund["legalName"], + "isin_number": fund.get("isin"), + "security_id": fund["secId"], + "data": {"list_info": fund}, + } + for fund in self.api_response["rows"] + ] + + def load(self): + """ + Load the data into the database + """ + + create_count = 0 + update_count = 0 + for data_dict in self.transformed_data: + try: + mf = self.model.objects.get(isin_number=data_dict["isin_number"]) + mf.data.update(data_dict["data"]) + mf.save() + update_count += 1 + except self.model.DoesNotExist: + mf = self.model(**data_dict) + mf.save() + create_count += 1 + + logger.info( + "Created %s records; Updated %s records", create_count, update_count + ) + + +class MFQuote(DataClient): + model = MutualFund + + # Monring Star get quote url + api_url = f"https://{settings.MORNINGSTAR_HOST}/etf/get-quote" + + def __init__(self, isin) -> None: + self.api_response = None # {"quotes": None, "holdings": None} + self.transformed_data = None + self.isin = isin + self.mf = self.model.objects.get(isin_number=self.isin) + + def extract(self): + + logger.info(f"Calling Morningstar Quote API for quotes with isin {self.isin}") + querystring = {"securityId": self.mf.security_id} + + response = requests.get( + self.api_url, headers=settings.MORNINGSTAR_API_HEADERS, params=querystring + ) + + # Check if the request was successful (status code 200) + if response.status_code == 200: + # Parse JSON response + self.api_response = response.json() + else: + logger.info(f"API response: %s", response.status_code) + response.raise_for_status() + + def load(self): + self.mf.data.update({"quotes": self.transformed_data}) + self.mf.save() + logger.info(f"Successfully stored data of quotes for {self.mf.fund_name}") + + +class MFHoldings(DataClient): + model = MutualFund + api_url = f"https://{settings.MORNINGSTAR_HOST}/etf/portfolio/get-holdings" + + def __init__(self, isin) -> None: + self.api_response = None + self.transformed_data = None + self.isin = isin + self.mf = self.model.objects.get(isin_number=self.isin) + + def extract(self): + + querystring = {"securityId": self.mf.security_id} + + response = requests.get( + self.api_url, headers=settings.MORNINGSTAR_API_HEADERS, params=querystring + ) + + # Check if the request was successful (status code 200) + if response.status_code == 200: + # Parse JSON response + self.api_response = response.json() + else: + logger.info(f"received status code {response.status_code} for {self.isin}") + logger.debug(response.content) + response.raise_for_status() + + def load(self): + self.mf.data.update({"holdings": self.transformed_data}) + self.mf.save() + logger.info(f"Successfully stored data of holdings for {self.mf.fund_name}") + + +class MFRiskMeasures(DataClient): + model = MutualFund + api_url = ( + f"https://{settings.MORNINGSTAR_HOST}/etf/risk/get-risk-volatility-measures" + ) + + def __init__(self, isin) -> None: + self.api_response = None + self.isin = isin + self.mf = self.model.objects.get(isin_number=self.isin) + + def extract(self): + + querystring = {"securityId": self.mf.security_id} + + response = requests.get( + self.api_url, headers=settings.MORNINGSTAR_API_HEADERS, params=querystring + ) + + # Check if the request was successful (status code 200) + if response.status_code == 200: + # Parse JSON response + self.api_response = response.json() + else: + logger.info(response.json()) + response.raise_for_status() + + def load(self): + self.mf.data.update({"risk_measures": self.transformed_data}) + self.mf.save() + logger.info( + f"Successfully stored data of risk measures for {self.mf.fund_name}" + ) + + +class MFRanking(DataClient): + + api_url = MONEYCONTROL_TOPFUNDS_URL + model = MutualFund + + def __init__(self) -> None: + self.api_response = None + self.transformed_data = None + + def extract(self) -> None: + """ + Fetches the top mutual funds from MoneyControl website based on their returns and + returns a tuple containing lists of fund names, fund types, CRISIL ranks, + INF numbers, and AUM data of top mutual funds. + """ + super().extract() + + logger.info("Fetching top mutual funds from MoneyControl website") + response = requests.get(self.api_url) + + # Check if the request was successful (status code 200) + response.raise_for_status() + + soup = BeautifulSoup(response.text, "html.parser") + + # Find all rows containing fund information + fund_rows = soup.find_all("tr", class_=lambda x: x and "INF" in x) + logger.info("Found %s rows", len(fund_rows)) + + fund_details = [] + + # Extract fund name from each row of sponsored funds + for row in fund_rows: + columns = row.find_all("td") + fund_name = columns[0].text.strip() + fund_type = columns[2].text.strip() + crisil_rank = columns[3].text.strip() + aum = columns[4].text.strip() + isin_number = row["class"][0] + + fund_details.append( + { + "fund_name": fund_name, + "fund_type": fund_type, + "crisil_rank": crisil_rank, + "isin_number": isin_number, + "aum": aum, + } + ) + + self.api_response = fund_details + + def load(self) -> None: + """ + Load the data into the database + """ + + # clear the rank field + MutualFund.objects.exclude(rank=None).update(rank=None) + + for rank, fund_details in enumerate(self.transformed_data, 1): + mf = MutualFund.objects.get(isin_number=fund_details["isin_number"]) + mf.crisil_rank = ( + fund_details["crisil_rank"] if fund_details["crisil_rank"] != "-" else 0 + ) + mf.rank = rank + mf.aum = float(fund_details["aum"].replace(",", "")) + mf.save() + logger.info( + f"Updated {rank=} {mf.fund_name} | {fund_details=} {fund_details['crisil_rank']=} {fund_details['aum']=}" + ) diff --git a/core/clients/stock.py b/core/clients/stock.py new file mode 100644 index 0000000000000000000000000000000000000000..c59c96527f0db7c9b5243efdde38ec884f0542d1 --- /dev/null +++ b/core/clients/stock.py @@ -0,0 +1,206 @@ +""" +This module contains the client classes for retrieving stock data from various sources. +""" + +import time +import logging + +import requests +from bs4 import BeautifulSoup + +from django.conf import settings +from core.models import Stock +from data_pipeline.interfaces.api_client import DataClient +from core.constants import MONEYCONTROL_TOPSTOCKS_URL + +logger = logging.getLogger(__name__) + + +class StockRankings(DataClient): + """ """ + + model = Stock + api_url = MONEYCONTROL_TOPSTOCKS_URL + + def __init__(self) -> None: + self.api_response = None + self.transformed_data = None + + def extract(self) -> None: + + logger.info("Fetching data from %s", self.api_url) + with requests.Session() as session: + try: + response = session.get(self.api_url) + response.raise_for_status() # Raise an HTTPError for bad responses (4xx or 5xx) + except requests.exceptions.RequestException as e: + logger.error(f"Error fetching data from {self.api_url}: {e}") + return + + soup = BeautifulSoup(response.text, "html.parser") + + # Find the table containing stock information + table = soup.find("table", {"id": "indicesTable"}) + if not table: + logger.warning( + "Table with id 'indicesTable' not found for stock during scraping ranks." + ) + raise Exception( + "Table with id 'indicesTable' not found for stock during scraping ranks." + ) + + data = [] + rows = table.find_all("tr") + logger.info(f"Found {len(rows)} rows in the table") + isin_fails = 0 + # Extract data from each row + for idx, row in enumerate(rows, start=1): + columns = row.find_all("td") + isin_number = None + if columns: + link = columns[0].find("a").get("href") if columns[0] else None + if link is not None: + try: + logger.info(f"Fetching stock details from link {link}") + response = session.get(link) + time.sleep(2) + response.raise_for_status() + soup = BeautifulSoup(response.text, "html.parser") + isin_element = soup.select_one( + 'li.clearfix span:contains("ISIN") + p' + ) + if isin_element: + isin_number = isin_element.get_text(strip=True) + else: + isin_fails += 1 + logger.warning(f"ISIN not found for link {link}") + except requests.exceptions.RequestException as e: + logger.exception( + f"Error fetching ISIN from link {link}: {e}" + ) + data.append( + { + "name": columns[0].get_text(strip=True), + "ltp": columns[1].get_text(strip=True), + "link": link, + "volume": columns[4].get_text(strip=True), + "percentage_change": columns[2].get_text(strip=True), + "price_change": columns[3].get_text(strip=True), + "rank": idx, + "isin_number": isin_number, + } + ) + logger.info(f"ISIN not found for {isin_fails} stocks out of {len(rows)}") + + self.api_response = data + + def load(self) -> None: + """ + Load the data into the database + """ + logger.info("Loading ranking data into the database...") + # clear the rank field + Stock.objects.exclude(rank=None).update(rank=None) + + for rank, stock_details in enumerate(self.transformed_data, 1): + try: + stock = Stock.objects.get(isin_number=stock_details["isin_number"]) + except Stock.DoesNotExist: + logger.warning( + f"No matching stock found for ISIN: {stock_details['isin_number']} creating new object..." + ) + stock = Stock.objects.create(data={"stock_rank": stock_details}) + + else: + stock.data.update({"stock_rank": stock_details}) + + stock.name = stock_details["name"] + stock.ltp = stock_details["ltp"] + stock.percentage_change = stock_details["percentage_change"] + stock.price_change = stock_details["price_change"] + stock.link = stock_details["link"] + stock.volume = stock_details["volume"] + stock.isin_number = stock_details["isin_number"] + stock.rank = stock_details["rank"] + stock.save() + + logger.info( + f"Saved {rank=} {stock.name} | {stock_details=} {stock_details['isin_number']=}" + ) + + +class StockDetails(DataClient): + """ + Retrieves and updates stock details from the Morningstar API. + """ + + model = Stock + api_url = f"https://{settings.MORNINGSTAR_HOST}/stock/get-detail" + + def __init__(self, perf_id: str, isin_number: str) -> None: + """ + Initializes the StockDetails object. + + Args: + perf_id (str): Performance ID of the stock. + isin_number (str): ISIN number of the stock. + """ + if not perf_id: + raise ValueError("Performance ID cannot be empty.") + if not isin_number: + raise ValueError("ISIN number cannot be empty.") + + self.api_response = {"details": None} + self.perf_id = perf_id + self.isin_number = isin_number + + def _request(self) -> requests.Response: + + querystring = {"PerformanceId": self.perf_id} + return requests.get( + self.api_url, + headers=settings.MORNINGSTAR_API_HEADERS, + params=querystring, + ) + + def extract(self) -> None: + """ + Extracts stock details from the Morningstar API. + """ + + response = self._request() + + requests_count = 1 + while response.status_code != 200: + if response.status_code == 429: + + logger.info( + f"API response: %s. Waiting for %s secs", + response.status_code, + 30 * requests_count, + ) + time.sleep(30 * requests_count) + response = self._request() + if requests_count > 3: + logger.warning( + f"API response: %s. Max retries reached", response.status_code + ) + break + requests_count += 1 + + else: + self.api_response["details"] = response.json() + logger.info(f"API response: %s", response.status_code) + + def load(self) -> None: + """ + Loads the retrieved stock details into the database. + """ + + stock = Stock.objects.filter(isin_number=self.isin_number).first() + if stock is None: + logger.warning(f"No matching stock found for ISIN: {self.isin_number}") + return + stock.data = self.transformed_data + stock.save() + logger.info(f"Successfully stored data for {stock.isin_number}.") diff --git a/core/constants.py b/core/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..2ca9452678c89f27821630877a459d8314d9c707 --- /dev/null +++ b/core/constants.py @@ -0,0 +1,4 @@ +MONEYCONTROL_TOPFUNDS_URL = "https://www.moneycontrol.com/mutual-funds/performance-tracker/returns/large-cap-fund.html" +MONEYCONTROL_TOPSTOCKS_URL = "https://www.moneycontrol.com/markets/indian-indices/changeTableData?deviceType=web&exName=N&indicesID=49&selTab=o&subTabOT=d&subTabOPL=cl&selPage=marketTerminal&classic=true" +TOPFUNDS_COUNT = 30 +STOCKS_MAX_RANK = 1000 diff --git a/core/cron.py b/core/cron.py new file mode 100644 index 0000000000000000000000000000000000000000..10a27efcae3383e3fa25a873ef97254cd1a45c33 --- /dev/null +++ b/core/cron.py @@ -0,0 +1,9 @@ +from data_pipeline import MFList + + +def store_mutual_funds(): + mf_list = MFList() + mf_list.extract() + mf_list.transform() + mf_list.load() + print("Stored successfully") diff --git a/core/management/commands/text2sql_eval.py b/core/management/commands/text2sql_eval.py new file mode 100644 index 0000000000000000000000000000000000000000..26afccf1ff963265b727a9139e7df6cac40b3611 --- /dev/null +++ b/core/management/commands/text2sql_eval.py @@ -0,0 +1,50 @@ +import logging +import time +import csv +from django.core.management.base import BaseCommand +from core.text2sql.handler import QueryDataHandler +from core.text2sql.prompt import get_prompt +from core.text2sql.eval_queries import queries + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = "Morningstar API to save the JSON response to a file which contains secIds with other details" + + def handle(self, *args, **options) -> None: + t1 = time.perf_counter() + q=[] + count =1 + for query in queries[26:]: + print("count: ", query["Query Number"]) + prompt = get_prompt(query["Query Description"]) + logger.info(f"Prompt: {prompt}") + generated_query, data = QueryDataHandler().get_data_from_query(prompt) + print(f"Description: {query['Query Description']}, Query: {query.get('SQL Statement')}, Generated: {generated_query} ") + q.append({ + "Query Number": query["Query Number"], + "Complexity Level": query["Complexity Level"], + "Description": query["Query Description"], + "Query": query.get("SQL Statement", "-"), + "Generated": generated_query, + }) + count+=1 + time.sleep(1) + csv_file_path = 'queries_data.csv' + + # Writing data to CSV + with open(csv_file_path, 'w', newline='', encoding='utf-8') as csv_file: + fieldnames = q[0].keys() + print(fieldnames) + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + + # Write the header + writer.writeheader() + + # Write the data + writer.writerows(q) + + print(f'Data has been written to {csv_file_path}.') + self.stdout.write(f"Time taken for evaluation: {time.perf_counter() - t1}") + + diff --git a/core/management/commands/update_mf.py b/core/management/commands/update_mf.py new file mode 100644 index 0000000000000000000000000000000000000000..7bdc3884614090d862ffeb9293f5b3d5a1c02d1e --- /dev/null +++ b/core/management/commands/update_mf.py @@ -0,0 +1,92 @@ +""" +This command is used to fetch all the data relating to mutual funds and save it to the database. +""" + +import logging +import time + +from django.core.management.base import BaseCommand +from core.constants import MONEYCONTROL_TOPSTOCKS_URL +from core.models import MutualFund +from core.clients.mutual_fund import ( + MFList, + MFQuote, + MFHoldings, + MFRiskMeasures, + MFRanking, +) +from core.clients.stock import StockDetails, StockRankings + +logger = logging.getLogger(__name__) + + +def get_funds_details() -> None: + """ + Get the details of the top 30 mutual funds and store them in the database. + """ + + t1 = time.perf_counter() + mutual_funds = MutualFund.objects.exclude(rank=None).order_by("rank")[:30] + logger.info(f"{mutual_funds=}") + for mf in mutual_funds: + # fetching the quotes data from the morningstar api and storing it in the database + MFQuote(mf.isin_number).run() + time.sleep(2) + # fetching the holdings data from the morningstar api and storing it in the database + MFHoldings(mf.isin_number).run() + time.sleep(2) + + # fetching the risk measures data from the morningstar api and storing it in the database + MFRiskMeasures(mf.isin_number).run() + time.sleep(2) + + logger.info("Time taken: %s", time.perf_counter() - t1) + + +def get_stock_details() -> None: + """ + Retrieves stock details for the top 30 mutual funds and updates the database. + """ + count = 0 + t1 = time.perf_counter() + mutual_funds = MutualFund.objects.exclude(rank=None).order_by("rank")[:30] + + for mf in mutual_funds: + + try: + holdings = ( + mf.data["holdings"].get("equityHoldingPage", {}).get("holdingList", []) + ) + except KeyError: + logger.warning("KeyError for holdings on Mutual Fund %s", mf.isin_number) + + for holding in holdings: + performance_id = holding.get("performanceId") + isin = holding.get("isin") + + if not performance_id or not isin: + logger.warning("Missing performanceId or isin for Mutual Fund %s", isin) + MFHoldings(mf.isin_number).run() + + stock_details = StockDetails(performance_id, isin) + stock_details.run() + count += 1 + + logger.info("Processed count: %s", count) + logger.info("Time taken by stock details: %s", time.perf_counter() - t1) + + +class Command(BaseCommand): + help = "Morningstar API to save the JSON response to a file which contains secIds with other details" + + def handle(self, *args, **options) -> None: + t1 = time.perf_counter() + try: + MFList().run() + MFRanking().run() + get_funds_details() + StockRankings().run() + get_stock_details() + except Exception as e: + logger.exception(e) + self.stdout.write(f"Time taken by: {time.perf_counter() - t1}") diff --git a/core/mfrating/score_calculator.py b/core/mfrating/score_calculator.py new file mode 100644 index 0000000000000000000000000000000000000000..a5720ff21e28e5a5fbd1fb0e11746b8527286241 --- /dev/null +++ b/core/mfrating/score_calculator.py @@ -0,0 +1,235 @@ +""" +This module defines a class, MFRating, which provides methods for calculating +the weighted rating and overall score for mutual funds based on various parameters. + +""" +import logging +from typing import List, Dict, Any +import numpy as np +from django.db.models import Max, Min +from core.models import MutualFund, Stock + + +logger = logging.getLogger(__name__) + + +class MFRating: + """ + This class provides methods for calculating the weighted stock rank rating and overall score for mutual funds based on various parameters. + """ + + def __init__(self, max_rank: int = 1000) -> None: + self.max_rank = max_rank + self.scores = { + "stock_ranking_score": [10], + "crisil_rank_score": [10], + "churn_score": [10], + "sharperatio_score": [10], + "expenseratio_score": [10], + "aum_score": [10], + "alpha_score": [10], + "beta_score": [10], + } + + def get_weighted_score(self, values: List[float]) -> float: + """ + Calculates the weighted rating based on the weights and values provided. + """ + weights = [] + values = [] + for _, (weight, score) in self.scores.items(): + weights.append(weight) + values.append(score) + + return np.average(values, weights=weights) + + def get_rank_rating(self, stock_ranks: List[int]) -> List[float]: + """ + Calculates the rank rating based on the stock ranks and the maximum rank. + """ + return [ + (self.max_rank - (rank if rank else self.max_rank)) / self.max_rank + for rank in stock_ranks + ] + + def get_overall_score(self, **kwargs) -> float: + """ + It returns the overall weighted score for mutual funds based on various parameters. + + """ + + stock_rankings = self.get_rank_rating(kwargs.get("stock_rankings")) + # what np.average do? + # Multiply each element in the stock_rankings array by its corresponding weights, then Sum up the results, then divide by the sum of the weights. + # data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + # weights = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1] + # + # Multiply each element in the data array by its corresponding weight: + # [1*10, 2*9, 3*8, 4*7, 5*6, 6*5, 7*4, 8*3, 9*2, 10*1] + # [10, 18, 24, 28, 30, 30, 28, 24, 18, 10] + # + # Sum up the results: + # 10 + 18 + 24 + 28 + 30 + 30 + 28 + 24 + 18 + 10 = 220 + # + # Sum up the weights: + # 10 + 9 + 8 + 7 + 6 + 5 + 4 + 3 + 2 + 1 = 55 + # + # Divide the sum of the weighted elements by the sum of the weights: + # 220 / 55 = 4.0 + self.scores["stock_ranking_score"].append( + np.average(stock_rankings, weights=kwargs.get("stock_weights")) + ) + self.scores["alpha_score"].append(kwargs.get("alpha", 0) / 100) + self.scores["beta_score"].append((2 - kwargs.get("beta", 2)) / 2) + self.scores["crisil_rank_score"].append( + (kwargs.get("crisil_rank_score", 0)) / 5 + ) + self.scores["churn_score"].append(kwargs.get("churn_rate", 0) / 100) + self.scores["sharperatio_score"].append(kwargs.get("sharpe_ratio", 0) / 100) + self.scores["expenseratio_score"].append(kwargs.get("expense_ratio", 0) / 100) + max_aum, min_aum, aum = kwargs.get("aum_score", (1, 0, 0)) + self.scores["aum_score"].append((aum - min_aum) / (max_aum - min_aum)) + # Calculate the overall rating using weighted sum + + return self.get_weighted_score(self.scores) + + +class MutualFundScorer: + def __init__(self) -> None: + self.mf_scores = [] + + def _get_stock_ranks(self, isin_ids: List[str]) -> List[int]: + """Get stock ranks based on ISIN ids.""" + + return list( + Stock.objects.filter(isin_number__in=isin_ids) + .order_by("rank") + .values_list("rank", "isin_number") + ) + + def _get_mutual_funds(self) -> List[MutualFund]: + """Get a list of top 30 mutual funds based on rank.""" + + return MutualFund.objects.exclude(rank=None).order_by("rank")[:30] + + def _get_risk_measure( + self, risk_measures: Dict[str, Any], key: str, year: str + ) -> float: + """ + Get value of the specified key from the risk_measures dictionary for the given year. + """ + try: + value = risk_measures.get(year, {}).get(key, 0) + return float(value) + except (TypeError, ValueError): + return 0 + + def _get_most_non_null_key(self, key, mutual_funds): + """ + Get the year with the maximum number of non-None values for the specified key + within the given mutual funds. + """ + year_counts = { + "for15Year": 0, + "for10Year": 0, + "for5Year": 0, + "for3Year": 0, + "for1Year": 0, + } + + for mf in mutual_funds: + risk_measures = mf.data["risk_measures"].get("fundRiskVolatility", {}) + + for year in year_counts: + if risk_measures.get(year, {}).get(key) is not None: + year_counts[year] += 1 + + most_non_null_year = max(year_counts, key=year_counts.get) + return most_non_null_year + + def get_scores(self) -> List[Dict[str, Any]]: + """Calculate scores for mutual funds and return the results.""" + + logger.info("Calculating scores for mutual funds...") + max_aum = MutualFund.objects.exclude(rank=None).aggregate(max_price=Max("aum"))[ + "max_price" + ] + min_aum = MutualFund.objects.exclude(rank=None).aggregate(min_price=Min("aum"))[ + "min_price" + ] + mutual_funds = self._get_mutual_funds() + + # Get the year with the maximum number of non-None values for sharpeRatio, alpha and beta + sharpe_ratio_year = self._get_most_non_null_key("sharpeRatio", mutual_funds) + alpha_year = self._get_most_non_null_key("alpha", mutual_funds) + beta_year = self._get_most_non_null_key("beta", mutual_funds) + for mf in mutual_funds: + mf_rating = MFRating( + max_rank=1000, + ) + logger.info(f"Processing mutual fund: %s", mf.fund_name) + holdings = ( + mf.data.get("holdings", {}) + .get("equityHoldingPage", {}) + .get("holdingList", []) + ) + portfolio_holding_weights = { + holding.get("isin"): ( + holding.get("weighting") if holding.get("weighting") else 0 + ) + for holding in holdings + if holding.get("isin") + } + stock_ranks_and_weights = [ + (rank, portfolio_holding_weights[isin]) + for rank, isin in self._get_stock_ranks( + portfolio_holding_weights.keys() + ) + ] + stock_ranks, stock_weights = zip(*stock_ranks_and_weights) + sharpe_ratio = self._get_risk_measure( + mf.data["risk_measures"].get("fundRiskVolatility", {}), + "sharpeRatio", + sharpe_ratio_year, + ) + alpha = self._get_risk_measure( + mf.data["risk_measures"].get("fundRiskVolatility", {}), + "alpha", + alpha_year, + ) + beta = self._get_risk_measure( + mf.data["risk_measures"].get("fundRiskVolatility", {}), + "beta", + beta_year, + ) + overall_score = mf_rating.get_overall_score( + stock_rankings=stock_ranks, + stock_weights=stock_weights, + churn_rate=mf.data["quotes"]["lastTurnoverRatio"] + if mf.data["quotes"].get("lastTurnoverRatio") + else 0, + sharpe_ratio=sharpe_ratio, + expense_ratio=mf.data["quotes"]["expenseRatio"], + crisil_rank_score=mf.crisil_rank, + aum_score=(max_aum, min_aum, mf.aum), + alpha=alpha, + beta=beta, + ) + + self.mf_scores.append( + { + "isin": mf.isin_number, + "name": mf.fund_name, + "rank": mf.rank, + "sharpe_ratio": round(sharpe_ratio, 4), + "churn_rate": mf.data["quotes"].get("lastTurnoverRatio", 0), + "expense_ratio": mf.data["quotes"].get("expenseRatio", 0), + "aum": mf.aum, + "alpha": round(alpha, 4), + "beta": round(beta, 4), + "crisil_rank": mf.crisil_rank, + "overall_score": round(overall_score, 4), + } + ) + logger.info("Finished calculating scores.") + return sorted(self.mf_scores, key=lambda d: d["overall_score"], reverse=True) diff --git a/core/middleware.py b/core/middleware.py new file mode 100644 index 0000000000000000000000000000000000000000..4d8bc432e14d9d2b47ce3020985324c7a9408c28 --- /dev/null +++ b/core/middleware.py @@ -0,0 +1,55 @@ +import logging +import traceback + +from django.http import JsonResponse + + +logger = logging.getLogger(__name__) + + +class ExceptionMiddleware: + """ + Middleware to catch exceptions and handle them with appropriate logging and JSON response. + """ + + def __init__(self, get_response): + """ + Initializes the ExceptionMiddleware with the provided get_response function. + """ + self.get_response = get_response + + def __call__(self, request): + """ + Process the request and call the next middleware or view function in the chain. + """ + response = self.get_response(request) + return response + + def process_exception(self, request, exception): + """ + Called when a view function raises an exception. + """ + error_type = exception.__class__.__name__ + error_message = exception.args + logger.info(f"Error Type: {error_type} | Error Message: {error_message}") + logger.debug("Request Details: %s", request.__dict__) + logger.exception(traceback.format_exc()) + + if isinstance(exception, KeyError): + status_code = 400 + message = f"Please Add Valid Data For {error_message[0]}" + error = "BAD_REQUEST" + elif isinstance(exception, AttributeError): + status_code = 500 + message = "Something Went Wrong. Please try again." + error = "SOMETHING_WENT_WRONG" + elif isinstance(exception, TypeError): + status_code = 500 + message = "Something Went Wrong. Please try again." + error = "SOMETHING_WENT_WRONG" + else: + status_code = 500 + message = "Something Went Wrong. Please try again." + error = "SOMETHING_WENT_WRONG" + + return JsonResponse({"message": message, "error": error}, status=status_code) diff --git a/core/migrations/0001_initial.py b/core/migrations/0001_initial.py new file mode 100644 index 0000000000000000000000000000000000000000..bab40c244694e0ea0aa2b8a946080c3005a108a2 --- /dev/null +++ b/core/migrations/0001_initial.py @@ -0,0 +1,41 @@ +# Generated by Django 4.2 on 2023-12-05 10:58 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="MutualFund", + fields=[ + ("created_at", models.DateTimeField(auto_now_add=True, null=True)), + ("updated_at", models.DateTimeField(auto_now=True, null=True)), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("fund_house", models.CharField(max_length=200)), + ( + "isin_number", + models.CharField(max_length=50, null=True, unique=True), + ), + ("security_id", models.CharField(max_length=50, unique=True)), + ("data", models.JSONField(null=True)), + ("rank", models.IntegerField(null=True, unique=True)), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/core/migrations/0002_rename_fund_house_mutualfund_fund_name.py b/core/migrations/0002_rename_fund_house_mutualfund_fund_name.py new file mode 100644 index 0000000000000000000000000000000000000000..663def04ccde4ddbdcc76c73d131dfc0d216a3e9 --- /dev/null +++ b/core/migrations/0002_rename_fund_house_mutualfund_fund_name.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-12-06 05:44 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0001_initial"), + ] + + operations = [ + migrations.RenameField( + model_name="mutualfund", + old_name="fund_house", + new_name="fund_name", + ), + ] diff --git a/core/migrations/0002_stock.py b/core/migrations/0002_stock.py new file mode 100644 index 0000000000000000000000000000000000000000..7885f15b002112079909d350cd68ef9c09d84533 --- /dev/null +++ b/core/migrations/0002_stock.py @@ -0,0 +1,41 @@ +# Generated by Django 4.2 on 2023-12-06 13:59 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="Stock", + fields=[ + ("created_at", models.DateTimeField(auto_now_add=True, null=True)), + ("updated_at", models.DateTimeField(auto_now=True, null=True)), + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("name", models.CharField(max_length=200)), + ("ltp", models.CharField(max_length=50, null=True)), + ("percentage_change", models.CharField(max_length=50, null=True)), + ("price_change", models.CharField(max_length=50, null=True)), + ("link", models.URLField(max_length=50, null=True)), + ("volume", models.CharField(max_length=50, null=True)), + ("data", models.JSONField(null=True)), + ("rank", models.IntegerField(null=True, unique=True)), + ], + options={ + "abstract": False, + }, + ), + ] diff --git a/core/migrations/0003_alter_mutualfund_fund_name.py b/core/migrations/0003_alter_mutualfund_fund_name.py new file mode 100644 index 0000000000000000000000000000000000000000..21b57a357227f6cd7a79bf0de6852db8fbb567b1 --- /dev/null +++ b/core/migrations/0003_alter_mutualfund_fund_name.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-12-12 11:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0002_rename_fund_house_mutualfund_fund_name"), + ] + + operations = [ + migrations.AlterField( + model_name="mutualfund", + name="fund_name", + field=models.CharField(max_length=200, unique=True), + ), + ] diff --git a/core/migrations/0003_alter_stock_link.py b/core/migrations/0003_alter_stock_link.py new file mode 100644 index 0000000000000000000000000000000000000000..135b58f29d2a60bb6d814bd5d5011e9f624efaa7 --- /dev/null +++ b/core/migrations/0003_alter_stock_link.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-12-06 14:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0002_stock"), + ] + + operations = [ + migrations.AlterField( + model_name="stock", + name="link", + field=models.URLField(null=True), + ), + ] diff --git a/core/migrations/0004_stock_isin_number.py b/core/migrations/0004_stock_isin_number.py new file mode 100644 index 0000000000000000000000000000000000000000..e8013baef4f8496a036ad74fc50f171d23e08d38 --- /dev/null +++ b/core/migrations/0004_stock_isin_number.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-12-07 06:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0003_alter_stock_link"), + ] + + operations = [ + migrations.AddField( + model_name="stock", + name="isin_number", + field=models.CharField(max_length=50, null=True, unique=True), + ), + ] diff --git a/core/migrations/0005_merge_20231211_0610.py b/core/migrations/0005_merge_20231211_0610.py new file mode 100644 index 0000000000000000000000000000000000000000..b8795d840a87946eeeaa06227bc42b229852e720 --- /dev/null +++ b/core/migrations/0005_merge_20231211_0610.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2 on 2023-12-11 06:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0002_rename_fund_house_mutualfund_fund_name"), + ("core", "0004_stock_isin_number"), + ] + + operations = [] diff --git a/core/migrations/0006_mutualfund_crisil_rank.py b/core/migrations/0006_mutualfund_crisil_rank.py new file mode 100644 index 0000000000000000000000000000000000000000..0c80eb6b89b6faadd6f3be8af93c11fcc25aba46 --- /dev/null +++ b/core/migrations/0006_mutualfund_crisil_rank.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-12-14 12:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0005_merge_20231211_0610"), + ] + + operations = [ + migrations.AddField( + model_name="mutualfund", + name="crisil_rank", + field=models.IntegerField(null=True), + ) + ] diff --git a/core/migrations/0007_merge_20231214_1924.py b/core/migrations/0007_merge_20231214_1924.py new file mode 100644 index 0000000000000000000000000000000000000000..a83e22015e0361b550a8bbdc674c0d723b67f311 --- /dev/null +++ b/core/migrations/0007_merge_20231214_1924.py @@ -0,0 +1,13 @@ +# Generated by Django 4.2 on 2023-12-14 19:24 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0003_alter_mutualfund_fund_name"), + ("core", "0006_mutualfund_crisil_rank"), + ] + + operations = [] diff --git a/core/migrations/0008_mutualfund_aum.py b/core/migrations/0008_mutualfund_aum.py new file mode 100644 index 0000000000000000000000000000000000000000..102b73236baff00765f478eebe307649ad9fe337 --- /dev/null +++ b/core/migrations/0008_mutualfund_aum.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-12-14 19:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0007_merge_20231214_1924"), + ] + + operations = [ + migrations.AddField( + model_name="mutualfund", + name="aum", + field=models.FloatField(null=True), + ), + ] diff --git a/core/migrations/0009_mutualfund_expense_ratio_mutualfund_return_m12_and_more.py b/core/migrations/0009_mutualfund_expense_ratio_mutualfund_return_m12_and_more.py new file mode 100644 index 0000000000000000000000000000000000000000..fef24f7162a2a4539467f8d31372a3d7c6ac848e --- /dev/null +++ b/core/migrations/0009_mutualfund_expense_ratio_mutualfund_return_m12_and_more.py @@ -0,0 +1,106 @@ +# Generated by Django 4.2 on 2024-01-09 12:19 + +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0008_mutualfund_aum"), + ] + + operations = [ + migrations.AddField( + model_name="mutualfund", + name="expense_ratio", + field=models.FloatField(blank=True, null=True), + ), + migrations.AddField( + model_name="mutualfund", + name="return_m12", + field=models.FloatField(blank=True, null=True), + ), + migrations.CreateModel( + name="MFVolatility", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "year", + models.CharField( + choices=[ + (1, "for1Year"), + (3, "for3Year"), + (5, "for5Year"), + (10, "for10Year"), + (15, "for15Year"), + ], + max_length=100, + ), + ), + ("alpha", models.FloatField(blank=True, null=True)), + ("beta", models.FloatField(blank=True, null=True)), + ("sharpe_ratio", models.FloatField(blank=True, null=True)), + ("standard_deviation", models.FloatField(blank=True, null=True)), + ( + "mutual_fund", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.mutualfund", + ), + ), + ], + ), + migrations.CreateModel( + name="MFHoldings", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("isin_number", models.CharField(max_length=20)), + ("security_id", models.CharField(max_length=20)), + ("sector", models.CharField(max_length=50)), + ("country", models.CharField(max_length=50)), + ("currency", models.CharField(max_length=20)), + ("weighting", models.FloatField(blank=True, null=True)), + ("sector_code", models.CharField(max_length=20)), + ("holding_type", models.CharField(max_length=20)), + ("market_value", models.FloatField(blank=True, null=True)), + ( + "stock_rating", + models.CharField(blank=True, max_length=10, null=True), + ), + ("total_assets", models.FloatField(blank=True, null=True)), + ("currency_name", models.CharField(max_length=50)), + ("maturity_date", models.DateField(blank=True, null=True)), + ("security_name", models.CharField(max_length=100)), + ("security_type", models.CharField(max_length=10)), + ("holding_type_id", models.CharField(max_length=1)), + ("number_of_shares", models.FloatField(blank=True, null=True)), + ("one_year_return", models.FloatField(blank=True, null=True)), + ("nav", models.FloatField(blank=True, null=True)), + ( + "mutual_fund", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.mutualfund", + ), + ), + ], + ), + ] diff --git a/core/migrations/0010_remove_mfholdings_nav_mutualfund_nav.py b/core/migrations/0010_remove_mfholdings_nav_mutualfund_nav.py new file mode 100644 index 0000000000000000000000000000000000000000..7e40983af39ac673a8d398dffd416831df463f08 --- /dev/null +++ b/core/migrations/0010_remove_mfholdings_nav_mutualfund_nav.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2 on 2024-01-09 12:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0009_mutualfund_expense_ratio_mutualfund_return_m12_and_more"), + ] + + operations = [ + migrations.RemoveField( + model_name="mfholdings", + name="nav", + ), + migrations.AddField( + model_name="mutualfund", + name="nav", + field=models.FloatField(blank=True, null=True), + ), + ] diff --git a/core/migrations/0011_remove_mfholdings_maturity_date_and_more.py b/core/migrations/0011_remove_mfholdings_maturity_date_and_more.py new file mode 100644 index 0000000000000000000000000000000000000000..cc80b54c1534d0cda46ede3743fce527ba5ea6a1 --- /dev/null +++ b/core/migrations/0011_remove_mfholdings_maturity_date_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2 on 2024-01-09 12:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0010_remove_mfholdings_nav_mutualfund_nav"), + ] + + operations = [ + migrations.RemoveField( + model_name="mfholdings", + name="maturity_date", + ), + migrations.AlterField( + model_name="mfholdings", + name="holding_type_id", + field=models.CharField(max_length=10), + ), + ] diff --git a/core/migrations/0012_alter_mfholdings_isin_number.py b/core/migrations/0012_alter_mfholdings_isin_number.py new file mode 100644 index 0000000000000000000000000000000000000000..ff5a70ae9c4c3f48e5e6aa045236e442a9eccb08 --- /dev/null +++ b/core/migrations/0012_alter_mfholdings_isin_number.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2024-01-09 13:06 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0011_remove_mfholdings_maturity_date_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="mfholdings", + name="isin_number", + field=models.CharField(blank=True, max_length=20, null=True), + ), + ] diff --git a/core/migrations/0013_alter_mfholdings_country_alter_mfholdings_currency_and_more.py b/core/migrations/0013_alter_mfholdings_country_alter_mfholdings_currency_and_more.py new file mode 100644 index 0000000000000000000000000000000000000000..0887c8353f3eb6b2b617778c14f039f83fad9dff --- /dev/null +++ b/core/migrations/0013_alter_mfholdings_country_alter_mfholdings_currency_and_more.py @@ -0,0 +1,63 @@ +# Generated by Django 4.2 on 2024-01-09 13:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0012_alter_mfholdings_isin_number"), + ] + + operations = [ + migrations.AlterField( + model_name="mfholdings", + name="country", + field=models.CharField(blank=True, max_length=50, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="currency", + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="currency_name", + field=models.CharField(blank=True, max_length=50, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="holding_type", + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="holding_type_id", + field=models.CharField(blank=True, max_length=10, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="sector", + field=models.CharField(blank=True, max_length=50, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="sector_code", + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="security_id", + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="security_name", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="security_type", + field=models.CharField(blank=True, max_length=10, null=True), + ), + ] diff --git a/core/migrations/0014_alter_mfholdings_currency_and_more.py b/core/migrations/0014_alter_mfholdings_currency_and_more.py new file mode 100644 index 0000000000000000000000000000000000000000..b2c8fb7cc763c2e49a09b7174a4a2c47fbd2871d --- /dev/null +++ b/core/migrations/0014_alter_mfholdings_currency_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 4.2 on 2024-01-09 13:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0013_alter_mfholdings_country_alter_mfholdings_currency_and_more"), + ] + + operations = [ + migrations.AlterField( + model_name="mfholdings", + name="currency", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="currency_name", + field=models.CharField(blank=True, max_length=150, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="holding_type", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="holding_type_id", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="sector_code", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="security_type", + field=models.CharField(blank=True, max_length=100, null=True), + ), + migrations.AlterField( + model_name="mfholdings", + name="stock_rating", + field=models.CharField(blank=True, max_length=100, null=True), + ), + ] diff --git a/core/migrations/0015_rename_security_name_mfholdings_holding_name_and_more.py b/core/migrations/0015_rename_security_name_mfholdings_holding_name_and_more.py new file mode 100644 index 0000000000000000000000000000000000000000..85769f6a16b5952a0e54216e2b152422f15cabd7 --- /dev/null +++ b/core/migrations/0015_rename_security_name_mfholdings_holding_name_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2 on 2024-01-09 14:10 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0014_alter_mfholdings_currency_and_more"), + ] + + operations = [ + migrations.RenameField( + model_name="mfholdings", + old_name="security_name", + new_name="holding_name", + ), + migrations.RemoveField( + model_name="mfholdings", + name="security_type", + ), + ] diff --git a/core/migrations/__init__.py b/core/migrations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/core/models.py b/core/models.py new file mode 100644 index 0000000000000000000000000000000000000000..929819ca2178655abe689bbcb8e1164268ee19ea --- /dev/null +++ b/core/models.py @@ -0,0 +1,100 @@ +import uuid +from django.db import models, connection + + +class BaseModel(models.Model): + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + created_at = models.DateTimeField(auto_now_add=True, null=True, blank=True) + updated_at = models.DateTimeField(auto_now=True, null=True, blank=True) + + class Meta: + abstract = True + + +class MutualFund(BaseModel): + """ + This model will store the mutual fund data + """ + + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + fund_name = models.CharField(max_length=200, unique=True) + isin_number = models.CharField(max_length=50, unique=True, null=True) + security_id = models.CharField(max_length=50, unique=True) + data = models.JSONField(null=True) + rank = models.IntegerField(unique=True, null=True) + crisil_rank = models.IntegerField(null=True) + aum = models.FloatField(null=True) + expense_ratio = models.FloatField(null=True, blank=True) + return_m12 = models.FloatField(null=True, blank=True) + nav = models.FloatField(null=True, blank=True) + + @staticmethod + def execute_raw_sql_query(sql_query): + with connection.cursor() as cursor: + cursor.execute(sql_query) + columns = [col[0] for col in cursor.description] + results = [dict(zip(columns, row)) for row in cursor.fetchall()] + + return results + + @classmethod + def execute_query(cls, query): + try: + return cls.execute_raw_sql_query(query) + except Exception as e: + return [] + +class Stock(BaseModel): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=200) + ltp = models.CharField(max_length=50, null=True) + percentage_change = models.CharField(max_length=50, null=True) + price_change = models.CharField(max_length=50, null=True) + link = models.URLField(max_length=200, null=True) + volume = models.CharField(max_length=50, null=True) + data = models.JSONField(null=True) + isin_number = models.CharField(max_length=50, unique=True, null=True) + rank = models.IntegerField(unique=True, null=True) + + +class MFHoldings(models.Model): + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + isin_number = models.CharField(max_length=20, null=True, blank=True) + security_id = models.CharField(max_length=20, null=True, blank=True) + sector = models.CharField(max_length=50, null=True, blank=True) + country = models.CharField(max_length=50, null=True, blank=True) + currency = models.CharField(max_length=100, null=True, blank=True) + weighting = models.FloatField(null=True, blank=True) + sector_code = models.CharField(max_length=100, null=True, blank=True) + holding_type = models.CharField(max_length=100, null=True, blank=True) + market_value = models.FloatField(null=True, blank=True) + stock_rating = models.CharField(max_length=100, null=True, blank=True) + total_assets = models.FloatField(null=True, blank=True) + currency_name = models.CharField(max_length=150, null=True, blank=True) + holding_name = models.CharField(max_length=100, null=True, blank=True) + holding_type = models.CharField(max_length=100, null=True, blank=True) + holding_type_id = models.CharField(max_length=100, null=True, blank=True) + number_of_shares = models.FloatField(null=True, blank=True) + one_year_return = models.FloatField(null=True, blank=True) + mutual_fund = models.ForeignKey(MutualFund, on_delete=models.CASCADE) + + def __str__(self): + return f"{self.ticker} - {self.securityName}" + + +class MFVolatility(models.Model): + VOLATILITY_CHOICES = ( + (1, "for1Year"), + (3, "for3Year"), + (5, "for5Year"), + (10, "for10Year"), + (15, "for15Year"), + ) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + mutual_fund = models.ForeignKey(MutualFund, on_delete=models.CASCADE) + year = models.CharField(max_length=100, choices=VOLATILITY_CHOICES) + alpha = models.FloatField(null=True, blank=True) + beta = models.FloatField(null=True, blank=True) + sharpe_ratio = models.FloatField(null=True, blank=True) + standard_deviation = models.FloatField(null=True, blank=True) diff --git a/core/test_models.py b/core/test_models.py new file mode 100644 index 0000000000000000000000000000000000000000..76e14f7ef86021685e4218a2efa4329c7713673c --- /dev/null +++ b/core/test_models.py @@ -0,0 +1,42 @@ +from django.test import TestCase +from core.models import MutualFund + + +class TestMutualFund(TestCase): + def test_model_creation(self): + # Create a new MutualFund instance + mutual_fund1 = MutualFund( + fund_name="Test Fund 1", + isin_number="123456789012", + security_id="MST01234", + data={ + "details": { + "legalName": "Test Fund 1", + "isin": "123456789012", + "secId": "MST01234", + } + }, + ) + + # Save the MutualFund instance to the database + mutual_fund1.save() + + # Check if the MutualFund instance was saved successfully + self.assertEqual(MutualFund.objects.count(), 1) + + mutual_fund2 = MutualFund( + fund_name="Test Fund 2", + isin_number="9876543210", + security_id="MST56789", + data={ + "details": { + "legalName": "Test Fund 2", + "isin": "9876543210", + "secId": "MST56789", + } + }, + ) + mutual_fund2.save() + + self.assertNotEqual(mutual_fund1.id, mutual_fund2.id) + self.assertEqual(MutualFund.objects.count(), 2) diff --git a/core/tests/__init__.py b/core/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/core/tests/data.py b/core/tests/data.py new file mode 100644 index 0000000000000000000000000000000000000000..8c470c564f987d0b24187403466944cc6bb6791f --- /dev/null +++ b/core/tests/data.py @@ -0,0 +1,339 @@ +test_data = { + 1: { + "quotes": {"expenseRatio": 0.023399999999999997, "lastTurnoverRatio": 0.1568}, + "holdings": { + "equityHoldingPage": { + "pageSize": 100, + "totalPage": 1, + "pageNumber": 1, + "holdingList": [ + { + "isin": "INE280A01028", + "weighting": 9.3, + }, + { + "isin": "INE002A01018", + "weighting": 7.1, + }, + {"isin": "INE090A01021", "weighting": 2.4}, + {"isin": "INE040A01034", "weighting": 1.1}, + ], + }, + }, + "list_info": {"isin": "MF123"}, + "risk_measures": { + "cur": "INR", + "fundName": "Testing fund 2", + "indexName": "Morningstar India GR INR", + "categoryName": "Large-Cap", + "fundRiskVolatility": { + "endDate": "2023-11-30T06:00:00.000", + "for1Year": { + "beta": 1.002, + "alpha": 4.464, + "rSquared": 98.8, + "sharpeRatio": 0.877, + "standardDeviation": 11.355, + }, + "for3Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for5Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for10Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for15Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "bestFitIndexName": None, + "forLongestTenure": None, + "bestFitBetaFor3Year": None, + "primaryIndexNameNew": "S&P BSE 100 India TR INR", + "bestFitAlphaFor3Year": None, + "bestFitRSquaredFor3Year": None, + }, + }, + }, + 2: { + "quotes": {"expenseRatio": 0.023399999999999997, "lastTurnoverRatio": 0.5158}, + "holdings": { + "equityHoldingPage": { + "pageSize": 100, + "totalPage": 1, + "pageNumber": 1, + "holdingList": [ + {"isin": "INE040A01034", "weighting": 7.6}, + {"isin": "INE154A01025", "weighting": 1.1}, + {"isin": "INE018A01030", "weighting": 3.2}, + {"isin": "INE154A01025", "weighting": 3.1}, + ], + }, + }, + "list_info": {"isin": "MF123"}, + "risk_measures": { + "cur": "INR", + "fundName": "Testing fund 2", + "indexName": "Morningstar India GR INR", + "categoryName": "Large-Cap", + "fundRiskVolatility": { + "endDate": "2023-11-30T06:00:00.000", + "for1Year": { + "beta": 0.994, + "alpha": 0.337, + "rSquared": 98.8, + "sharpeRatio": 0.341, + "standardDeviation": 11.355, + }, + "for3Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for5Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for10Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for15Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "bestFitIndexName": None, + "forLongestTenure": None, + "bestFitBetaFor3Year": None, + "primaryIndexNameNew": "S&P BSE 100 India TR INR", + "bestFitAlphaFor3Year": None, + "bestFitRSquaredFor3Year": None, + }, + }, + }, + 3: { + "quotes": {"expenseRatio": 0.0106, "lastTurnoverRatio": 0.3109}, + "holdings": { + "equityHoldingPage": { + "pageSize": 100, + "totalPage": 1, + "pageNumber": 1, + "holdingList": [ + {"isin": "INE280A01028", "weighting": 11.2}, + {"isin": "INE090A01021", "weighting": 7.1}, + {"isin": "INE040A01034", "weighting": 2.4}, + ], + }, + }, + "list_info": {"isin": "MF123"}, + "risk_measures": { + "cur": "INR", + "fundName": "Testing fund 2", + "indexName": "Morningstar India GR INR", + "categoryName": "Large-Cap", + "fundRiskVolatility": { + "endDate": "2023-11-30T06:00:00.000", + "for1Year": { + "beta": 0.954, + "alpha": -0.645, + "rSquared": 98.8, + "sharpeRatio": 0.251, + "standardDeviation": 11.355, + }, + "for3Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for5Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for10Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for15Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "bestFitIndexName": None, + "forLongestTenure": None, + "bestFitBetaFor3Year": None, + "primaryIndexNameNew": "S&P BSE 100 India TR INR", + "bestFitAlphaFor3Year": None, + "bestFitRSquaredFor3Year": None, + }, + }, + }, + 4: { + "quotes": {"expenseRatio": 0.0158, "lastTurnoverRatio": 0.5451}, + "holdings": { + "equityHoldingPage": { + "pageSize": 100, + "totalPage": 1, + "pageNumber": 1, + "holdingList": [ + {"isin": "INE280A01028", "weighting": 10.2}, + {"isin": "INE002A01018", "weighting": 9.2}, + ], + }, + }, + "list_info": {"isin": "MF123"}, + "risk_measures": { + "cur": "INR", + "fundName": "Testing fund 2", + "indexName": "Morningstar India GR INR", + "categoryName": "Large-Cap", + "fundRiskVolatility": { + "endDate": "2023-11-30T06:00:00.000", + "for1Year": { + "beta": 1.137, + "alpha": 5.675, + "rSquared": 98.8, + "sharpeRatio": 0.735, + "standardDeviation": 11.355, + }, + "for3Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for5Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for10Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for15Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "bestFitIndexName": None, + "forLongestTenure": None, + "bestFitBetaFor3Year": None, + "primaryIndexNameNew": "S&P BSE 100 India TR INR", + "bestFitAlphaFor3Year": None, + "bestFitRSquaredFor3Year": None, + }, + }, + }, + 5: { + "quotes": {"expenseRatio": 0.0216, "lastTurnoverRatio": 0.2025}, + "holdings": { + "equityHoldingPage": { + "pageSize": 100, + "totalPage": 1, + "pageNumber": 1, + "holdingList": [ + {"isin": "INE002A01018", "weighting": 13.2}, + {"isin": "INE090A01021", "weighting": 7.4}, + {"isin": "INE040A01034", "weighting": 3.4}, + ], + }, + }, + "list_info": {"isin": "MF123"}, + "risk_measures": { + "cur": "INR", + "fundName": "Testing fund 2", + "indexName": "Morningstar India GR INR", + "categoryName": "Large-Cap", + "fundRiskVolatility": { + "endDate": "2023-11-30T06:00:00.000", + "for1Year": { + "beta": 1.041, + "alpha": 0.521, + "rSquared": 98.8, + "sharpeRatio": 0.354, + "standardDeviation": 11.355, + }, + "for3Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for5Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for10Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "for15Year": { + "beta": None, + "alpha": None, + "rSquared": None, + "sharpeRatio": None, + "standardDeviation": None, + }, + "bestFitIndexName": None, + "forLongestTenure": None, + "bestFitBetaFor3Year": None, + "primaryIndexNameNew": "S&P BSE 100 India TR INR", + "bestFitAlphaFor3Year": None, + "bestFitRSquaredFor3Year": None, + }, + }, + }, +} diff --git a/core/tests/test_scores.py b/core/tests/test_scores.py new file mode 100644 index 0000000000000000000000000000000000000000..bf53e5f553df6c0c1ee99f417cb82092efb80427 --- /dev/null +++ b/core/tests/test_scores.py @@ -0,0 +1,101 @@ +from django.test import TestCase +from core.models import MutualFund, Stock +from core.mfrating.score_calculator import MutualFundScorer, MFRating +from core.tests.data import test_data + + +class MutualFundScorerTestCase(TestCase): + """ + Test case for the MutualFundScorer class to test scores. + """ + + def setUp(self): + self.stock_data = [ + {"isin_number": "INE040A01034", "rank": 10}, + {"isin_number": "INE090A01021", "rank": 21}, + {"isin_number": "INE002A01018", "rank": 131}, + {"isin_number": "INE154A01025", "rank": 99}, + {"isin_number": "INE018A01030", "rank": 31}, + {"isin_number": "INE280A01028", "rank": 2}, + ] + + self.mutual_fund_data = [ + { + "isin_number": "ISIN1", + "fund_name": "Testing Fund 1", + "rank": 1, + "aum": 837.3, + "crisil_rank": 4, + "security_id": "SEC1", + "data": test_data[1], + }, + { + "isin_number": "ISIN2", + "fund_name": "Testing Fund 2", + "rank": 2, + "aum": 210.3, + "crisil_rank": 1, + "security_id": "SEC2", + "data": test_data[2], + }, + { + "isin_number": "ISIN3", + "fund_name": "Testing Fund 3", + "rank": 3, + "aum": 639.3, + "crisil_rank": 3, + "security_id": "SEC3", + "data": test_data[3], + }, + { + "isin_number": "ISIN4", + "fund_name": "Testing Fund 4", + "rank": 4, + "aum": 410.3, + "crisil_rank": 2, + "security_id": "SEC4", + "data": test_data[4], + }, + { + "isin_number": "ISIN5", + "fund_name": "Testing Fund 5", + "rank": 5, + "aum": 1881.3, + "crisil_rank": 5, + "security_id": "SEC5", + "data": test_data[5], + }, + ] + + self.create_stock_objects() + self.create_mutual_fund_objects() + self.mf_scorer = MutualFundScorer() + + def create_stock_objects(self): + """ + Create stock objects using the predefined stock data. + """ + self.stock_objects = [Stock.objects.create(**data) for data in self.stock_data] + + def create_mutual_fund_objects(self): + """ + Create mutual fund objects using the predefined mutual fund data. + """ + self.mutual_fund_objects = [ + MutualFund.objects.create(**data) for data in self.mutual_fund_data + ] + + def test_get_scores_returns_sorted_list(self): + """ + Test whether the get_scores method returns a sorted list of scores. + """ + scores = self.mf_scorer.get_scores() + self.assertEqual(len(scores), 5) + self.assertEqual( + scores, sorted(scores, key=lambda x: x["overall_score"], reverse=True) + ) + expected_scores = [0.4263, 0.3348, 0.2962, 0.2447, 0.2101] + for i, expected_score in enumerate(expected_scores): + self.assertAlmostEqual( + scores[i]["overall_score"], expected_score, delta=1e-4 + ) diff --git a/core/text2sql/__init__.py b/core/text2sql/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/core/text2sql/eval_queries.py b/core/text2sql/eval_queries.py new file mode 100644 index 0000000000000000000000000000000000000000..bc290bc3a9644f788eaf933ffc7862b28e780ea4 --- /dev/null +++ b/core/text2sql/eval_queries.py @@ -0,0 +1,86 @@ +queries = [ + {"Query Number": 1, "Complexity Level": "Simple", "Query Description": "Retrieve all mutual funds and their names", + "SQL Statement": "SELECT id, fund_name FROM core_mutualfund;"}, + {"Query Number": 2, "Complexity Level": "Simple", "Query Description": "Get the total number of mutual funds", + "SQL Statement": "SELECT COUNT(*) FROM core_mutualfund;"}, + {"Query Number": 3, "Complexity Level": "Simple", "Query Description": "List all unique ISIN numbers in the mutual fund holdings", + "SQL Statement": "SELECT DISTINCT isin_number FROM core_mfholdings;"}, + {"Query Number": 4, "Complexity Level": "Simple", "Query Description": + "Find the mutual fund with the highest AUM (Assets Under Management)", "SQL Statement": "SELECT * FROM core_mutualfund ORDER BY aum DESC LIMIT 1;"}, + {"Query Number": 5, "Complexity Level": "Simple", "Query Description": "Retrieve the top 5 mutual funds with the highest one-year return", + "SQL Statement": "SELECT * FROM core_mutualfund ORDER BY return_m12 DESC LIMIT 5;"}, + {"Query Number": 6, "Complexity Level": "Medium", "Query Description": "List mutual funds with their holdings and respective sector codes", + "SQL Statement": "SELECT m.fund_name, h.sector, h.sector_code FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id;"}, + {"Query Number": 7, "Complexity Level": "Medium", "Query Description": "Find the average expense ratio for all mutual funds", + "SQL Statement": "SELECT AVG(expense_ratio) FROM core_mutualfund;"}, + {"Query Number": 8, "Complexity Level": "Medium", "Query Description": "Retrieve mutual funds with a specific country in their holdings", + "SQL Statement": "SELECT m.fund_name, h.country FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id WHERE h.country = 'USA';"}, + {"Query Number": 9, "Complexity Level": "Medium", "Query Description": + "List mutual funds with volatility metrics (alpha, beta, sharpe_ratio)", "SQL Statement": "SELECT m.fund_name, v.alpha, v.beta, v.sharpe_ratio FROM core_mutualfund m JOIN core_mfvolatility v ON m.id = v.mutual_fund_id;"}, + {"Query Number": 10, "Complexity Level": "Medium", "Query Description": + "Retrieve mutual funds with a NAV (Net Asset Value) greater than a specific value", "SQL Statement": "SELECT * FROM core_mutualfund WHERE nav > 100;"}, + {"Query Number": 11, "Complexity Level": "High", "Query Description": "Find the mutual fund with the highest total market value of holdings", + "SQL Statement": "SELECT m.fund_name, MAX(h.market_value) AS max_market_value FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id;"}, + {"Query Number": 12, "Complexity Level": "High", "Query Description": "List mutual funds with their average one-year return grouped by sector", + "SQL Statement": "SELECT h.sector, AVG(m.return_m12) AS avg_one_year_return FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id GROUP BY h.sector;"}, + {"Query Number": 13, "Complexity Level": "High", "Query Description": "Retrieve mutual funds with a specific stock rating in their holdings", + "SQL Statement": "SELECT m.fund_name, h.stock_rating FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id WHERE h.stock_rating = 'A';"}, + {"Query Number": 14, "Complexity Level": "High", "Query Description": "Find the mutual fund with the lowest standard deviation of volatility", + "SQL Statement": "SELECT m.fund_name, MIN(v.standard_deviation) AS min_standard_deviation FROM core_mutualfund m JOIN core_mfvolatility v ON m.id = v.mutual_fund_id;"}, + {"Query Number": 15, "Complexity Level": "High", "Query Description": "List mutual funds with the highest number of shares in their holdings", + "SQL Statement": "SELECT m.fund_name, MAX(h.number_of_shares) AS max_number_of_shares FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id;"}, + {"Query Number": 16, "Complexity Level": "More Complex", "Query Description": "Retrieve mutual funds and their holdings with a specific currency", + "SQL Statement": "SELECT m.fund_name, h.currency FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id WHERE h.currency = 'USD';"}, + {"Query Number": 17, "Complexity Level": "More Complex", "Query Description": "Find the mutual fund with the highest total market value across all holdings", + "SQL Statement": "SELECT m.fund_name, SUM(h.market_value) AS total_market_value FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id GROUP BY m.fund_name ORDER BY total_market_value DESC LIMIT 1;"}, + {"Query Number": 18, "Complexity Level": "More Complex", "Query Description": "List mutual funds with their average alpha and beta values for a specific year", + "SQL Statement": "SELECT m.fund_name, AVG(v.alpha) AS avg_alpha, AVG(v.beta) AS avg_beta FROM core_mutualfund m JOIN core_mfvolatility v ON m.id = v.mutual_fund_id WHERE v.year = '2023' GROUP BY m.fund_name;"}, + {"Query Number": 19, "Complexity Level": "More Complex", "Query Description": "Retrieve mutual funds with a specific holding type and its market value", + "SQL Statement": "SELECT m.fund_name, h.holding_type, h.market_value FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id WHERE h.holding_type = 'Equity';"}, + {"Query Number": 20, "Complexity Level": "More Complex", "Query Description": "List mutual funds with their rankings and corresponding CRISIL rankings", + "SQL Statement": "SELECT m.fund_name, m.rank, m.crisil_rank FROM core_mutualfund m;"}, + {"Query Number": 21, "Complexity Level": "More Complex", "Query Description": "Find the mutual fund with the highest average one-year return across all years", + "SQL Statement": "SELECT m.fund_name, AVG(m.return_m12) AS avg_one_year_return FROM core_mutualfund m GROUP BY m.fund_name ORDER BY avg_one_year_return DESC LIMIT 1;"}, + {"Query Number": 22, "Complexity Level": "More Complex", "Query Description": "Retrieve mutual funds with their top 3 holdings based on market value", + "SQL Statement": "SELECT m.fund_name, h.holding_name, h.market_value FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id ORDER BY h.market_value DESC LIMIT 3;"}, + {"Query Number": 23, "Complexity Level": "More Complex", "Query Description": "List mutual funds with their volatility metrics for a specific year", + "SQL Statement": "SELECT m.fund_name, v.year, v.alpha, v.beta, v.sharpe_ratio, v.standard_deviation FROM core_mutualfund m JOIN core_mfvolatility v ON m.id = v.mutual_fund_id WHERE v.year = '2022';"}, + {"Query Number": 24, "Complexity Level": "More Complex", "Query Description": "Find the mutual fund with the highest average market value per holding", + "SQL Statement": "SELECT m.fund_name, AVG(h.market_value) AS avg_market_value_per_holding FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id GROUP BY m.fund_name ORDER BY avg_market_value_per_holding DESC LIMIT 1;"}, + { + "Query Number": 25, + "Complexity Level": "More Complex", + "Query Description": "Retrieve mutual funds with their total assets and the corresponding volatility metrics", + "SQL Statement": "SELECT m.fund_name, h.total_assets, v.alpha, v.beta, v.sharpe_ratio, v.standard_deviation FROM core_mutualfund m JOIN core_mfholdings h ON m.id = h.mutual_fund_id JOIN core_mfvolatility v ON m.id = v.mutual_fund_id;" + }, + { + "Query Number": 26, + "Complexity Level": "More Complex", + "Query Description":"Retrieve mutual funds that have holdings in both technology and healthcare sectors, and provide a breakdown of their allocation percentages in each sector.", + "SQL Statement":"-" + }, + { + "Query Number": 27, + "Complexity Level": "More Complex", + "Query Description": "Retrieve Mutual Funds with Highest Average Return and Lowest Expense Ratio", + "SQL Statement":"-" + }, + { + "Query Number": 28, + "Complexity Level": "More Complex", + "Query Description":"Find Mutual Funds with Diversified Holdings", + "SQL Statement":"-" + }, + { + "Query Number": 29, + "Complexity Level": "More Complex", + "Query Description": "Identify Mutual Funds with Consistent Performance and High AUM", + "SQL Statement":"-" + }, + { + "Query Number": 30, + "Complexity Level": "More Complex", + "Query Description": "Calculate Weighted Average Return for Mutual Funds in a Specific Sector", + "SQL Statement":"-" + } +] diff --git a/core/text2sql/handler.py b/core/text2sql/handler.py new file mode 100644 index 0000000000000000000000000000000000000000..f578de6ba6ce61a80beff2acb62bb349bc2d7d04 --- /dev/null +++ b/core/text2sql/handler.py @@ -0,0 +1,26 @@ +import logging +from core.models import MutualFund +from core.text2sql.ml_models import Text2SQLModel + +logger = logging.getLogger(__name__) + + +class QueryDataHandler: + """ + A class for handling queries and fetching data using a Text2SQL model and Django models. + """ + + def __init__(self): + self.mutual_fund = MutualFund() + self.text2sql_model = Text2SQLModel() + + def get_data_from_query(self, prompt): + """ + Generates a PostgreSQL query using the Text2SQL model based on the input prompt + and retrieves data using Django models. + """ + # Use Text2SQL ML model to generate a PostgreSQL query + sql_query = self.text2sql_model.generate_query(prompt) + logger.info(f"SQL Query: {sql_query}") + # Use Django models to fetch data + return sql_query, self.mutual_fund.execute_query(sql_query) diff --git a/core/text2sql/ml_models.py b/core/text2sql/ml_models.py new file mode 100644 index 0000000000000000000000000000000000000000..d4eeeec58bd4d5dbb28bb6b5cf9c1108cef1b519 --- /dev/null +++ b/core/text2sql/ml_models.py @@ -0,0 +1,24 @@ +import replicate + + +class Text2SQLModel: + """ + A class representing a Text-to-SQL model for generating SQL queries from LLM. + """ + + def __init__(self): + pass + + def load_model(self): + """Loads the machine learning model for Text-to-SQL processing.""" + pass + + def generate_query(self, prompt): + output = replicate.run( + "ns-dev-sentience/sqlcoder:18bcabd866a64547daf3c6044cdebbd47a1f489571110087b80722848eb09398", + input={"prompt": prompt}, + ) + return ( + output.split("```PostgresSQL")[-1].split("```")[0].split(";")[0].strip() + + ";" + ) diff --git a/core/text2sql/prompt.py b/core/text2sql/prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..2463af18cbacb15044ce8d5b4685132f91766bdc --- /dev/null +++ b/core/text2sql/prompt.py @@ -0,0 +1,69 @@ +def get_prompt(question): + database_schema = """ +CREATE TABLE core_mutualfund ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + created_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP, + fund_name VARCHAR(200) UNIQUE, + isin_number VARCHAR(50) UNIQUE, + security_id VARCHAR(50) UNIQUE, + data JSONB, + rank INTEGER UNIQUE, + crisil_rank INTEGER, + aum DOUBLE PRECISION, + expense_ratio DOUBLE PRECISION, + nav DOUBLE PRECISION, + return_m12 DOUBLE PRECISION +); + +CREATE TABLE core_mfholdings ( + "id" uuid NOT NULL PRIMARY KEY, + "isin_number" varchar(20) NULL, + "security_id" varchar(20) NULL, + "sector" varchar(50) NULL, + "country" varchar(50) NULL, + "currency" varchar(100) NULL, + "weighting" double precision NULL, + "sector_code" varchar(100) NULL, + "holding_type" varchar(100) NULL, + "market_value" double precision NULL, + "stock_rating" varchar(100) NULL, + "total_assets" double precision NULL, + "currency_name" varchar(150) NULL, + "holding_name" varchar(100) NULL, + "holding_type" varchar(100) NULL, + "holding_type_id" varchar(100) NULL, + "number_of_shares" double precision NULL, + "one_year_return" double precision NULL, + "mutual_fund_id" uuid NOT NULL REFERENCES "core_mutualfund" ("id") DEFERRABLE INITIALLY DEFERRED +); + +CREATE TABLE core_mfvolatility ( + "id" uuid NOT NULL PRIMARY KEY, + "mutual_fund_id" uuid NOT NULL REFERENCES "core_mutualfund" ("id") DEFERRABLE INITIALLY DEFERRED, + "year" varchar(100) NOT NULL, + "alpha" double precision NULL, + "beta" double precision NULL, + "sharpe_ratio" double precision NULL, + "standard_deviation" double precision NULL +); + +-- core_mfvolatility.mutual_fund_id can be joined with core_mutualfund.id +-- core_mfholdings.mutual_fund_id can be joined with core_mutualfund.id + """ + + sql_prompt = f""" +Your task is to convert a question into a PostgresSQL query, given a database schema. + +###Task: +Generate a SQL query that answers the question `{question}`. + +### Database Schema: +This query will run on a database whose schema is represented below: +{database_schema} + +### Response: +Based on your instructions, here is the PostgresSQL query I have generated to answer the question `{question}`: +```PostgresSQL + """ + return sql_prompt diff --git a/core/urls.py b/core/urls.py new file mode 100644 index 0000000000000000000000000000000000000000..cdc46a1cbcbdbbf0ab7e50d324b498438bca2515 --- /dev/null +++ b/core/urls.py @@ -0,0 +1,7 @@ +from django.urls import path +from core.views import get_scores, get_mf_data + +urlpatterns = [ + path("mutual-fund-details/", get_scores, name="mutual-fund-details"), + path("get-mf-data/", get_mf_data, name="get-mf-data"), +] diff --git a/core/views.py b/core/views.py new file mode 100644 index 0000000000000000000000000000000000000000..5315e50574b8f6bde601143be5b7803d9083e9c8 --- /dev/null +++ b/core/views.py @@ -0,0 +1,30 @@ +""" + +""" +import logging +from django.http import JsonResponse +from core.mfrating.score_calculator import MutualFundScorer +from core.text2sql.handler import QueryDataHandler +from core.text2sql.prompt import get_prompt + +logger = logging.getLogger(__name__) + + +def get_scores(request): + """ + Retrieves scores for mutual funds based on various factors. + """ + data = MutualFundScorer().get_scores() + return JsonResponse({"status": "success", "data": data}, status=200) + + +def get_mf_data(request): + """ + Retrieves mutual fund data based on user query. + """ + query = request.GET.get("query", "") + logger.info(f"Query: {query}") + prompt = get_prompt(query) + logger.info(f"Prompt: {prompt}") + query, data = QueryDataHandler().get_data_from_query(prompt) + return JsonResponse({"status": "success", "query": query, "data": data}, status=200) diff --git a/data_pipeline/__init__.py b/data_pipeline/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data_pipeline/admin.py b/data_pipeline/admin.py new file mode 100644 index 0000000000000000000000000000000000000000..8c38f3f3dad51e4585f3984282c2a4bec5349c1e --- /dev/null +++ b/data_pipeline/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/data_pipeline/apps.py b/data_pipeline/apps.py new file mode 100644 index 0000000000000000000000000000000000000000..c24f759abb51856b7fa4cc9a66baa8636f4c45f2 --- /dev/null +++ b/data_pipeline/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class DataPipelineConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "data_pipeline" diff --git a/data_pipeline/interfaces/__init__.py b/data_pipeline/interfaces/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data_pipeline/interfaces/api_client.py b/data_pipeline/interfaces/api_client.py new file mode 100644 index 0000000000000000000000000000000000000000..37a3a420685be4d186dde241b437bbc353a0d81b --- /dev/null +++ b/data_pipeline/interfaces/api_client.py @@ -0,0 +1,36 @@ +""" +Interface for API clients +""" +from abc import ABC, abstractmethod + + +class DataClient(ABC): + """ + Abstract class for API clients + """ + + @abstractmethod + def extract(self): + """ + Extract data from API + """ + if self.api_url is None: + raise Exception("No API URL provided") + + def transform(self): + """ + Placeholder method for future transformation logic. + """ + self.transformed_data = self.api_response + + @abstractmethod + def load(self): + """ + Load data into target model + """ + raise Exception("No model to transform") + + def run(self) -> None: + self.extract() + self.transform() + self.load() diff --git a/data_pipeline/interfaces/test_api_client.py b/data_pipeline/interfaces/test_api_client.py new file mode 100644 index 0000000000000000000000000000000000000000..9e79ac86b19a6cdd97ccc4144b65d665e73d4ca9 --- /dev/null +++ b/data_pipeline/interfaces/test_api_client.py @@ -0,0 +1,35 @@ +""" + +""" +from django.test import TestCase +from data_pipeline.interfaces.api_client import DataClient + + +class TestDataClient(TestCase): + def test_extract_raises_exception_on_instantiation(self): + with self.assertRaises(TypeError): + data_client = DataClient() + + def test_extract_raises_exception_without_api_url(self): + class NewDataClientWithoutFunctionOverride(DataClient): + def __init__(self) -> None: + pass + + with self.assertRaises(TypeError): + data_client = NewDataClientWithoutFunctionOverride() + + def test_inherited_class(self): + class NewDataClientWith3FunctionOverride(DataClient): + def __init__(self) -> None: + pass + + def extract(self): + pass + + def transform(self): + pass + + def load(self): + pass + + data_client = NewDataClientWith3FunctionOverride() diff --git a/data_pipeline/migrations/__init__.py b/data_pipeline/migrations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data_pipeline/models.py b/data_pipeline/models.py new file mode 100644 index 0000000000000000000000000000000000000000..71a836239075aa6e6e4ecb700e9c42c95c022d91 --- /dev/null +++ b/data_pipeline/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/data_pipeline/tests/__init__.py b/data_pipeline/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/data_pipeline/tests/test_datapipeline.py b/data_pipeline/tests/test_datapipeline.py new file mode 100644 index 0000000000000000000000000000000000000000..ce7493f3be2bc90525c0c6e51572cbb27de5bd03 --- /dev/null +++ b/data_pipeline/tests/test_datapipeline.py @@ -0,0 +1,38 @@ +from django.test import TestCase +from unittest.mock import patch, Mock + +# from data_pipeline.api_helper import MFList + + +# class TestMFListExtract(TestCase): + +# @patch('requests.get') +# def test_extract_successful_response(self, mock_get): +# # Mock a successful response from the Morningstar API +# mock_get.return_value = Mock(status_code=200, json=lambda: {'rows': [{'legalName': 'Test Fund 1', 'isin': '123456789012', 'secId': 'MST01234'}]}) + +# # Create an instance of the MFList class +# mf_list = MFList() + +# # Call the extract method +# mf_list.extract() + +# # Assert that the API response was set +# self.assertIsNotNone(mf_list.api_response) + +# # Assert that the transformed data contains the extracted fund information +# self.assertEqual(mf_list.transformed_data, [{'fund_name': 'Test Fund 1', 'isin_number': '123456789012', 'security_id': 'MST01234'}]) + +# @patch('requests.get') +# def test_extract_unsuccessful_response(self, mock_get): +# # Mock an unsuccessful response from the Morningstar API +# mock_get.return_value = Mock(status_code=500, json=lambda: {'error': 'Internal Server Error'}) + +# # Create an instance of the MFList class +# mf_list = MFList() + +# # Call the extract method +# mf_list.extract() + +# # Assert that the API response was set +# self.assertIsNone(mf_list.api_response) diff --git a/data_pipeline/views.py b/data_pipeline/views.py new file mode 100644 index 0000000000000000000000000000000000000000..91ea44a218fbd2f408430959283f0419c921093e --- /dev/null +++ b/data_pipeline/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/manage.py b/manage.py new file mode 100755 index 0000000000000000000000000000000000000000..3ecec2e883d542dbdb8677380a7cf786dcebc7cc --- /dev/null +++ b/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + """Run administrative tasks.""" + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "stockfund.settings") + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == "__main__": + main() diff --git a/model/.dockerignore b/model/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..4522d57dc7d6a100de8c39a6039a4a125ac61c4d --- /dev/null +++ b/model/.dockerignore @@ -0,0 +1,17 @@ +# The .dockerignore file excludes files from the container build process. +# +# https://docs.docker.com/engine/reference/builder/#dockerignore-file + +# Exclude Git files +.git +.github +.gitignore + +# Exclude Python cache files +__pycache__ +.mypy_cache +.pytest_cache +.ruff_cache + +# Exclude Python virtual environment +/venv diff --git a/model/__init__.py b/model/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/model/cog.yaml b/model/cog.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c7623fb13d215ab249f599094fdd754a6202026 --- /dev/null +++ b/model/cog.yaml @@ -0,0 +1,29 @@ +# Configuration for Cog ⚙️ +# Reference: https://github.com/replicate/cog/blob/main/docs/yaml.md + +build: + # set to true if your model requires a GPU + gpu: true + + # a list of ubuntu apt packages to install + # system_packages: + # - "libgl1-mesa-glx" + # - "libglib2.0-0" + + # python version in the form '3.11' or '3.11.4' + python_version: "3.11.1" + + # a list of packages in the format == + python_packages: + - "transformers==4.36.2" + - "torch==1.13.0" + - "accelerate==0.25.0" + # - "torchvision==0.9.0" + + # commands run after the environment is setup + # run: + # - "echo env is ready!" + # - "echo another command if needed" + +# predict.py defines how predictions are run on your model +predict: "predict.py:Predictor" \ No newline at end of file diff --git a/model/predict.py b/model/predict.py new file mode 100644 index 0000000000000000000000000000000000000000..3441caeec81e78432af9a2e5875eef5028e75cba --- /dev/null +++ b/model/predict.py @@ -0,0 +1,57 @@ +# Prediction interface for Cog ⚙️ +# https://github.com/replicate/cog/blob/main/docs/python.md + +from cog import BasePredictor, Input +import torch +from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline +import argparse + + +class Predictor(BasePredictor): + def setup(self) -> None: + """Load the model into memory to make running multiple predictions efficient""" + # self.model = torch.load("./weights.pth") + model_name = "defog/sqlcoder-34b-alpha" + self.tokenizer = AutoTokenizer.from_pretrained(model_name) + self.model = AutoModelForCausalLM.from_pretrained( + model_name, + torch_dtype=torch.float16, + device_map="auto", + use_cache=True, + offload_folder="./.cache", + ) + + def predict( + self, + prompt: str = Input(description="Prompt to generate from"), + ) -> str: + """Run a single prediction on the model""" + # processed_input = preprocess(image) + # output = self.model(processed_image, scale) + # return postprocess(output) + + # make sure the model stops generating at triple ticks + # eos_token_id = tokenizer.convert_tokens_to_ids(["```"])[0] + eos_token_id = self.tokenizer.eos_token_id + pipe = pipeline( + "text-generation", + model=self.model, + tokenizer=self.tokenizer, + max_length=300, + do_sample=False, + num_beams=5, # do beam search with 5 beams for high quality results + ) + generated_query = ( + pipe( + prompt, + num_return_sequences=1, + eos_token_id=eos_token_id, + pad_token_id=eos_token_id, + )[0]["generated_text"] + .split("```sql")[-1] + .split("```")[0] + .split(";")[0] + .strip() + + ";" + ) + return generated_query diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..7fe417dcad4cf6cfd89cd647294072f230fdc01c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,93 @@ +altair==5.2.0 +anyio==4.0.0 +appdirs==1.4.4 +asgiref==3.7.2 +asttokens==2.4.1 +attrs==23.1.0 +beautifulsoup4==4.12.2 +blinker==1.7.0 +bs4==0.0.1 +cachetools==5.3.2 +certifi==2023.11.17 +cfgv==3.4.0 +charset-normalizer==3.3.2 +click==8.1.7 +decorator==5.1.1 +Deprecated==1.2.14 +distlib==0.3.7 +Django==4.2 +django-crontab==0.7.1 +executing==2.0.1 +filelock==3.13.1 +finnhub-python==2.4.19 +frozendict==2.3.8 +gitdb==4.0.11 +GitPython==3.1.40 +h11==0.14.0 +html5lib==1.1 +httpcore==1.0.2 +httpx==0.25.1 +identify==2.5.32 +idna==3.4 +importlib-metadata==6.11.0 +ipython==8.18.0 +jedi==0.19.1 +Jinja2==3.1.2 +jsonschema==4.20.0 +jsonschema-specifications==2023.11.2 +lxml==4.9.3 +markdown-it-py==3.0.0 +MarkupSafe==2.1.3 +matplotlib-inline==0.1.6 +mdurl==0.1.2 +mftool==2.7.1 +multitasking==0.0.11 +nodeenv==1.8.0 +numpy==1.26.2 +packaging==23.2 +pandas==2.1.3 +parso==0.8.3 +peewee==3.17.0 +pexpect==4.9.0 +Pillow==10.1.0 +platformdirs==3.11.0 +pre-commit==3.5.0 +prompt-toolkit==3.0.41 +protobuf==4.25.1 +psycopg2-binary==2.9.9 +ptyprocess==0.7.0 +pure-eval==0.2.2 +pyarrow==14.0.2 +pydeck==0.8.1b0 +Pygments==2.17.2 +python-dateutil==2.8.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +referencing==0.32.0 +requests==2.31.0 +rich==13.7.0 +rpds-py==0.15.2 +six==1.16.0 +smmap==5.0.1 +sniffio==1.3.0 +soupsieve==2.5 +sqlparse==0.4.4 +stack-data==0.6.3 +streamlit==1.29.0 +tenacity==8.2.3 +toml==0.10.2 +toolz==0.12.0 +tornado==6.4 +traitlets==5.13.0 +typing_extensions==4.9.0 +tzdata==2023.3 +tzlocal==5.2 +urllib3==2.1.0 +validators==0.22.0 +virtualenv==20.24.6 +watchdog==3.0.0 +wcwidth==0.2.12 +webencodings==0.5.1 +wrapt==1.16.0 +yfinance==0.2.32 +zipp==3.17.0 diff --git a/stockfund/__init__.py b/stockfund/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/stockfund/asgi.py b/stockfund/asgi.py new file mode 100644 index 0000000000000000000000000000000000000000..02f2ce8e404012aa02487950bcdf2034c3c6b4b9 --- /dev/null +++ b/stockfund/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for stockfund project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "stockfund.settings") + +application = get_asgi_application() diff --git a/stockfund/settings/__init__.py b/stockfund/settings/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b095fbdaaafd147fbd69ec0b89857f2bf82eaf61 --- /dev/null +++ b/stockfund/settings/__init__.py @@ -0,0 +1,6 @@ +from .base import * + +try: + from .local import * +except ImportError: + pass diff --git a/stockfund/settings/base.py b/stockfund/settings/base.py new file mode 100644 index 0000000000000000000000000000000000000000..e19ba517c54b585e1fced597d3e3228b3789b726 --- /dev/null +++ b/stockfund/settings/base.py @@ -0,0 +1,217 @@ +""" +Django settings for stockfund project. + +Generated by 'django-admin startproject' using Django 4.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" +import os +from pathlib import Path + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent.parent + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = "django-insecure-miq=zs#3kpf79%j$!bhq*++ho5nf5b!9ri(j(v*y%rw=we)1^b" + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] +DATA_UPLOAD_MAX_NUMBER_FIELDS = 50000 + +# Application definition + +INSTALLED_APPS = [ + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "core", + "django_crontab", + "data_pipeline", +] + +MIDDLEWARE = [ + "core.middleware.ExceptionMiddleware", + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", +] + +ROOT_URLCONF = "stockfund.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "stockfund.wsgi.application" + + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": os.environ["DB_NAME"], + "USER": os.environ["DB_USER"], + "PASSWORD": os.environ["DB_PASSWORD"], + "HOST": os.environ["DB_HOST"], + "PORT": "5432", + "TEST": { + "NAME": os.environ.get("TEST_DB_NAME", "mf_backend"), + }, + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", + }, + { + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = "en-us" + +TIME_ZONE = "UTC" + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = "static/" + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + +CRONJOBS = [ + ("0 0 1 * *", "core.cron.store_mutual_funds"), +] + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", + }, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", + }, + }, + "formatters": { + "verbose": { + "format": "{levelname} {asctime} {module}.{funcName} {lineno} - {message}", + "style": "{", + }, + "simple": { + "format": "{levelname} {message}", + "style": "{", + }, + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[{server_time}] {message}", + "style": "{", + }, + }, + "handlers": { + "console": { + "level": "INFO", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + }, + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", + }, + "mail_admins": { + "level": "ERROR", + "filters": ["require_debug_false"], + "class": "django.utils.log.AdminEmailHandler", + }, + "file": { + "level": "DEBUG", + "class": "logging.FileHandler", + "filename": os.path.join(BASE_DIR, "debug.log"), + "formatter": "verbose", + }, + }, + "loggers": { + "django": { + "handlers": ["console", "mail_admins"], + "level": "DEBUG", + }, + "django.server": { + "handlers": ["django.server"], + "level": "DEBUG", + "propagate": False, + }, + "core": { + "handlers": ["file"], + "level": "DEBUG", + "propagate": False, + }, + "*": { + "handlers": ["file"], + "level": "DEBUG", + "propagate": False, + }, + }, +} + +MORNINGSTAR_KEY = os.environ["MORNINGSTAR_KEY"] +MORNINGSTAR_HOST = "morning-star.p.rapidapi.com" + +MORNINGSTAR_API_HEADERS = { + "X-RapidAPI-Key": MORNINGSTAR_KEY, + "X-RapidAPI-Host": MORNINGSTAR_HOST, +} diff --git a/stockfund/urls.py b/stockfund/urls.py new file mode 100644 index 0000000000000000000000000000000000000000..0fff8cea9efcf509a14b279e61a22b9a4f43db61 --- /dev/null +++ b/stockfund/urls.py @@ -0,0 +1,23 @@ +""" +URL configuration for stockfund project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path, include + +urlpatterns = [ + path("admin/", admin.site.urls), + path("api/", include("core.urls")), +] diff --git a/stockfund/wsgi.py b/stockfund/wsgi.py new file mode 100644 index 0000000000000000000000000000000000000000..b3386832f728b66344db9fbb7afb5c5c273d53ef --- /dev/null +++ b/stockfund/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for stockfund project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "stockfund.settings") + +application = get_wsgi_application() diff --git a/tests/__initi__.py b/tests/__initi__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/data.py b/tests/data.py new file mode 100644 index 0000000000000000000000000000000000000000..415053c20efdb65d4270cccf6e5fce8e5abf11c2 --- /dev/null +++ b/tests/data.py @@ -0,0 +1,12 @@ +import os +import json + +from django.conf import settings + + +def get_mflist_data(): + with open( + os.path.join(settings.BASE_DIR, "tests/data/morningstar_mf_list.json") + ) as f: + data = json.load(f) + return data diff --git a/text2sql_ui.py b/text2sql_ui.py new file mode 100644 index 0000000000000000000000000000000000000000..bee99abebb0a4de45329a3a0072b33cb19832731 --- /dev/null +++ b/text2sql_ui.py @@ -0,0 +1,40 @@ +""" +UI for text2sql app +""" +import os +import pandas as pd +import requests +import streamlit as st + +# Streamlit app +st.set_page_config(layout="wide") + + +def main(): + st.title("Mutual Fund Text2SQL App") + + # Get user prompt from Streamlit UI + prompt = st.text_input("Enter your question here:") + + if st.button("Submit"): + + API_URL = f"{os.environ['SERVER_URL']}/api/get-mf-data/?query={prompt}" + response = requests.get(API_URL) + + if response.status_code != 200: + st.error("Error fetching data from the server.") + st.stop() + + df = pd.DataFrame(response.json()["data"]) + st.write("Query:", response.json()["query"]) + # st.markdown( + # "

Mutual Fund Data Analysis Tool

", + # unsafe_allow_html=True, + # ) + + # Display the DataFrame without scrolling and use the full page width + st.dataframe(df, width=10000, height=1000) + + +if __name__ == "__main__": + main() diff --git a/ui.py b/ui.py new file mode 100644 index 0000000000000000000000000000000000000000..19eac27845699bd91541cad9ac9294902ad57d90 --- /dev/null +++ b/ui.py @@ -0,0 +1,24 @@ +import os +import streamlit as st +import pandas as pd +import requests + +API_URL = f"{os.environ['SERVER_URL']}/api/mutual-fund-details/" +response = requests.get(API_URL) + +if response.status_code != 200: + st.error("Error fetching data from the server.") + st.stop() + +df = pd.DataFrame(response.json()["data"]) + +# Streamlit app +st.set_page_config(layout="wide") + +st.markdown( + "

Mutual Fund Data Analysis Tool

", + unsafe_allow_html=True, +) + +# Display the DataFrame without scrolling and use the full page width +st.dataframe(df, width=10000, height=1000)