Upload 2 files

app_stream.py

@@ -0,0 +1,187 @@
+import streamlit as st
+# import wikipedia
+from streamlit_lottie import st_lottie 
+import regex as re
+from streamlit_js_eval import streamlit_js_eval
+from common.utils import *
+from data_collector import *
+from langchain_community.llms import Ollama
+
+from langchain_community.llms import Ollama
+import pandas as pd
+import json
+st.set_page_config(layout="wide")
+screen_width = streamlit_js_eval(label="screen.width",js_expressions='screen.width')
+screen_height = streamlit_js_eval(label="screen.height",js_expressions='screen.height')
+
+
+condition_capture = st.session_state
+if 'schema' not in condition_capture:
+    condition_capture['schema'] = {}
+if 'prompt' not in condition_capture:
+    condition_capture['prompt'] = ""
+if  "count" not in condition_capture:
+    condition_capture['count'] = 0
+if "prev_schema" not in condition_capture:
+    condition_capture['prev_schema'] = {}
+if "textual_value" not in condition_capture:
+    condition_capture['textual_value'] = {}
+textual_value=None
+
+
+
+schema=condition_capture['schema']
+
+
+
+
+column1,column2 = st.columns(2)
+with column2:
+
+
+
+    if len(condition_capture['schema'])!=0 and len(condition_capture['textual_value'])==0:
+        # condition_capture['prev_schema'] = condition_capture['schema']
+        condition_capture['textual_value']=relevent_value(str(condition_capture['schema']).lower(),50)
+    if len(condition_capture['schema'])!=0:
+        html_page = condition_capture['textual_value'][1]
+        textual_value = condition_capture['textual_value'][0]
+        st.write("<br>",unsafe_allow_html=True)
+        
+        with st.container(border=True,height=int(screen_height/2.3)):
+            st.header("Wikipedia insights")
+            updated_schema = st.button("Start processing")
+            selector=st.empty()
+            write =st.empty()
+            start_page= selector.select_slider("Select a range of color wavelength",options=[i for i in html_page],key="start_page")
+            write.write(html_page[start_page],unsafe_allow_html=True)
+            
+            
+            
+            
+                
+# )
+            
+            
+                    
+
+
+        
+
+    
+with column1:
+    
+    if str(schema)!=str({}):
+        tabs = st.tabs(["Schema","Data Generation"])
+        with tabs[0]:
+            if str(schema)!=str({}):
+
+                schema_column1,schema_column2 = st.columns(2)
+                with schema_column1:
+                        edited_df = st.data_editor([str(i) for index,i in enumerate(schema)],hide_index=True,use_container_width=True,num_rows='dynamic',height=int(screen_height/3))
+                        
+                with schema_column2:
+                    number = st.number_input("Number of rows",min_value=1,max_value=1000,value=10)
+                    if number!=condition_capture['count'] and updated_schema:
+                        condition_capture['count'] = number
+                    
+                    
+                    with open("animation\\edit_file.json") as animate:
+                        url_json=json.load(animate)
+                    st_lottie(url_json,height = int(screen_height/3))
+                   
+
+        with tabs[1]:
+            with open("animation\\no data animation.json") as animate:
+                url_json=json.load(animate)
+            dataframe=st.empty()
+            
+            if condition_capture['count']==0:
+                st_lottie(url_json,height = int(screen_height/3))
+                
+            else:
+                smart_append=[]
+                if condition_capture['prev_schema'] != condition_capture['schema']:
+                    condition_capture['prev_schema'] = condition_capture['schema']
+                    condition_capture['current_append']={}
+                    
+                    for text_indexing,store in enumerate(actual_value(textual_value,schema)):
+                        dummy_value =dictionary_formatting(store)
+                        for keys in dummy_value:
+                            while len(dummy_value[keys])>=2:
+                                dummy_value[keys].pop(0)
+                        dummy_value = dictionary_formatting(dummy_value)
+
+                        if dummy_value != None:
+                            
+                            
+                            smart_append.append(dummy_value)
+                            print(dummy_value)
+                            for keys in dummy_value:
+                                if keys not in condition_capture['current_append']:
+                                    condition_capture['current_append'][str(keys)]=[]
+                                condition_capture['current_append'][str(keys)].append(str([i for i in dummy_value[keys]]))
+                            dataframe.dataframe(condition_capture['current_append'])
+                            
+
+                            if len(condition_capture['current_append'][[i for i in condition_capture['current_append']][-1]])>=condition_capture['count']:
+                                break
+
+                            # print(dummy_value)
+                            # if smart_check(dummy_value)!=True:
+                            #     smart_value=verification(dummy_value)
+                            # if statement(condition_capture['schema'],smart_value):
+                            #     st.dataframe(smart_value)
+                condition_capture['current_append']={}
+                if len(smart_append)==0:
+                    
+                    ranger=len(condition_capture['current_append'][[i for i in condition_capture['current_append']][0]])
+                    for indexing in range(ranger):
+                        working_dict = {}
+                        for j in condition_capture['current_append']:
+                        
+                            working_dict[j]=condition_capture['current_append'][j][indexing][0]
+                        smart_append.append(working_dict)
+                smart_movement =   sorting(smart_append)
+                
+                for keys in smart_movement:
+                    value=eval(keys)
+                    for keys in value:
+                        if keys not in condition_capture['current_append']:
+                            condition_capture['current_append'][str(keys)]=[]
+                        condition_capture['current_append'][str(keys)].append([str(i) for i in value[keys]])
+                dataframe.dataframe(condition_capture['current_append'])
+                for indexing,j in enumerate(smart_movement):
+                    try:
+                        # Convert string to dictionary
+                        dummy_value = eval(j)
+                        
+                        # Process dictionary values
+                        for key in dummy_value:
+                            while len(dummy_value[key]) >= 2:
+                                dummy_value[key].pop(0)
+                        
+                        # Format dictionary
+                        formatted = dictionary_formatting(dummy_value)
+                        print(formatted)
+                        # Verify and store result
+                        verification_result = verification(formatted) if formatted else None
+                        for j in verification_result:
+                            if j in condition_capture['current_append']:
+                                condition_capture['current_append'][j][indexing]=[str(i) for i in verification_result[j]]
+                        dataframe.dataframe(condition_capture['current_append'])
+                        
+                    except:
+                        pass
+
+
+  
+prompt = st.text_input(label="Please use prompt to generate data",value=condition_capture['prompt'])
+if prompt != str(condition_capture['prompt']):
+    
+    condition_capture['prompt'] = prompt
+    schema = schema_generator(prompt)
+    condition_capture['schema'] = schema
+    condition_capture['current_append']={}
+    
+    st.rerun()

data_collector.py

@@ -0,0 +1,399 @@
+
+import wikipedia
+import wikipediaapi
+import regex as re
+from sentence_transformers import SentenceTransformer,util
+from transformers import pipeline
+import requests
+
+# def consume_llm_api(prompt):
+#     """
+#     Sends a prompt to the LLM API and processes the streamed response.
+#     """
+#     url = "https://3c93-70-167-32-130.ngrok-free.app/api/llm-response"
+#     headers = {"Content-Type": "application/json"}
+#     payload = {"prompt": prompt,"extension":"1"}
+
+    
+#     print("Sending prompt to the LLM API...")
+#     response_ = requests.post(url, json=payload,verify=False)
+#     response_data = response_.json()
+#     return response_data['text']
+def consume_llm_api(prompt):
+    model = Ollama(model="llama3:latest", temperature=0.3)
+    return model.invoke(prompt)
+
+
+def relevent_value(long_query,count=3):
+    results = wikipedia.search(long_query,results=count)
+    
+    wiki_wiki = wikipediaapi.Wikipedia(user_agent='MyProjectName ([email protected])', language='en',extract_format=wikipediaapi.ExtractFormat.WIKI)
+    wiki_wiki_html = wikipediaapi.Wikipedia(user_agent='MyProjectName ([email protected])', language='en',extract_format=wikipediaapi.ExtractFormat.HTML)
+    values={}
+    html_values={}
+    for result in results:
+        page_py = wiki_wiki.page(result)
+        page_html = wiki_wiki_html.page(result)
+        html_values[result]=page_html.text
+
+        values[result]=page_py.text
+    return values,html_values
+
+
+from langchain_community.llms import Ollama
+model=Ollama(model="llama3:latest",temperature=0.3)
+agent_understanding = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+qa_model = pipeline('question-answering', model='deepset/roberta-base-squad2', tokenizer='deepset/roberta-base-squad2')
+
+# textual_value
+
+def construction_edit(textual_value,schema):
+    construction_prompt= textual_value+"\n"
+    construction_prompt+="Above is the generated text from wikipedia and below is the rule that has to be filled in the data. \n"
+    construction_prompt+="The data should be in the form of a dictionary and it must follow the following schema: \n"
+    construction_prompt+=str(schema)+"\n"
+    construction_prompt+="The length of each list of each key must be same in the generated data(mandatory)."+"\n"
+    construction_prompt+="No helper text like 'Here is the filled-in JSON schema based on the provided text' or 'Note: I've filled in the keys with relevant data' ."+ "\n"
+    construction_prompt+="The output must be a dictionary"+"\n"
+    constructed_text=consume_llm_api(construction_prompt)
+    return constructed_text
+
+def dictionary_check(construction_edit):
+    for keys in construction_edit:
+        if len(construction_edit[keys])==0:
+            return False
+    return True
+
+def actual_value(textual_value,schema):
+    for j in textual_value:
+        formatted_result = str(textual_value[j])+ "\n"
+        formatted_result += "Please fill the following schema with the relevant data from the text above."+ "\n"
+        formatted_result += "Here is the schema"+"\n"
+        formatted_result += str(schema)
+        formatted_result += "Please generate data according to schema and fill this template with your answers.\n"
+        formatted_result += "You have to fill each key with the relevant data from the text above."+ "\n"
+        formatted_result += "Please return the exact key value pair as the schema above. "+ "\n"
+        formatted_result += "No helper text like 'Here is the filled-in JSON schema based on the provided text' or 'Note: I've filled in the keys with relevant data' ."+ "\n"
+        formatted_result += "Only fill the keys that are in the schema."+ "\n"
+        formatted_result += "If you are not sure about the data, you can add 'Na'."+ "\n"
+        formatted_result += "It's an order you can not add any other text(e.g Here is the filled-in JSON schema) or note ."+ "\n"
+        formatted_result += "The length of each list of each key must be same in the generated data(mandatory)."+"\n"
+        raw_output = consume_llm_api(formatted_result)
+        try:
+            data=construction_edit(raw_output,schema)
+            json_object_match = re.search(r'\{(?:[^{}]|(?R))*\}', data)
+            access_value=eval(json_object_match.group())
+            for schema_key in schema:
+                if schema_key not in access_value:
+                    access_value[schema_key]=list(set())
+            for schema_key in access_value:
+                access_value[schema_key]=list(set(access_value[schema_key]))
+                access_value[schema_key]=list(set(access_value[schema_key])-set(["Na"]))
+            yield access_value
+            
+        except:
+            access_value=None
+        
+
+
+
+def context_data_relevancy(value,context):
+    researcher =  "You are a professional reasearcher from data ."+ "\n"
+    researcher += "You have to check can we fill some of the missing values in the "+str(value) + ". \n"  
+    researcher += "The possible part which available in the context has to be relevent with already present data"+ ". \n"
+    researcher += "from the context given below"+ ". \n"
+    researcher += context+ "\n"
+    researcher += "Be strict while thing of filling data"+ ". \n"
+    researcher += "Just return @yahoo@ if 90% possible else @NO@"+ ". \n"
+
+
+    result = consume_llm_api(researcher)
+    return result
+
+def agent_work_result(query,value):
+    agent_understanding = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+    query_embedding = agent_understanding.encode(query)
+    score1 = util.cos_sim(query_embedding,agent_understanding.encode("extract data for"))
+    score2 = util.cos_sim(query_embedding,agent_understanding.encode("append data in "))
+    score3 = util.cos_sim(query_embedding,agent_understanding.encode("check data"))
+
+    if score1 > score2 and score1 > score3:
+        # print("Extracting query:", query)
+        question = "search word ?"
+        result = qa_model(question=question, context=query)
+        result = result['answer']
+        print("Extracting query:", result)
+        wikisearch = relevent_value(result,3)
+        html_pages = wikisearch[1]
+        wikisearch = wikisearch[0]
+
+        for searches in wikisearch:
+            if "@yahoo@" in context_data_relevancy(value,wikisearch[searches]):
+                return wikisearch[searches]
+        return "No data found"
+    elif score2 > score1 and score2 > score3:
+        try:
+            print("Appending command:", query)
+            question1 = "which value we are adding to key ?"
+            result1 = qa_model(question=question1, context=query)
+            question2 = "In which key we are appending ?"
+            result2 = qa_model(question=question2, context=query)
+            result1 = result1['answer']
+            result2 = result2['answer']
+
+            if  len(value[result2])==0:
+                value[result2].append(result1)
+                return "Now you can fill the remaining columns"
+            else:
+                return "You are putting value in the same key column again not accepted."
+        except Exception as e:
+            return str(e)
+    else:
+        min_=0
+        max_=0
+        for keys in value:
+            
+            if len(value[keys])<min_:
+                min_=len(value[keys])
+            if len(value[keys])>max_:
+                max_=len(value[keys])
+        if min_==max_:
+            return "You dia a great job"
+        else:
+            return "Please append the data correctly so that the length of each key is same and data is also relevant"
+    
+
+def full_alignment(value):
+    for values in value:
+        if len(value[values])==0:
+            return False
+    return True
+
+def query_formatting(result):
+    values=result.split("\n")
+    if len(values)!=0:
+        values.pop(0)
+    return values
+def missing_value_completion(store,value):
+
+    filler_prompt = "Below is mentioned ajson data\n"
+    filler_prompt += str(value)+"\n"
+    filler_prompt += "you only need to find missing data from the mentioned context section."
+    filler_prompt += "You will return the results in below mentioned format.\n"
+    filler_prompt += "The output will be in json format."
+    filler_prompt += "context:\n"
+    
+    for search_key in store:
+        try:
+            fill_text = store[search_key]
+            response = consume_llm_api(filler_prompt+fill_text)
+        
+            json_object_match = re.search(r'\{(?:[^{}]|(?R))*\}', response)
+            access_value=eval(json_object_match.group())
+            for keys in value:
+                if len(value[keys])==0 and keys in access_value:
+                    value[keys].append(access_value[keys].pop(0))
+            print(value)
+            if full_alignment(value):
+                return value
+        except:
+            pass
+        
+
+
+
+def verification(value):
+    
+
+    validation_prompt = "Can you prepare a list of text(many as possible) that can be searched on google for filling(relevent data) the missing data below.\n"
+    validation_prompt += str(value)+"\n"
+    validation_prompt += "You need to prepare it by the following manner"
+    validation_prompt += "1. Mention it line by line.\n"
+    validation_prompt += "2. Please seperate it line by line.\n"
+    validation_prompt += "3. Headers are not required\n"
+    validation_prompt += "4. Please do not add any helper text example: Here is the required search queries , Here are the search queries .\n"
+    validation_prompt += "5. Please do not add any notes"
+    print("Searching for missing values")
+    result=query_formatting(consume_llm_api(validation_prompt))
+
+    for search_queries in result:
+        if len(search_queries)!=0:
+            print(search_queries)
+            store=relevent_value(search_queries)
+            html_pages = store[1]
+            store = store[0]
+            missing_value_completion(store,value)
+        if full_alignment(value):
+            return value
+
+            
+
+
+
+    return result
+
+def agent_data_prep(value,query):
+    end_result = ""
+    angent_earlier_income ="0"
+    pre_money_saving = "0"
+    mission = "First to fill most importent column \n"
+    while end_result!="You dia a great job":
+        
+        if full_alignment(value):
+            return value
+            
+
+        agent_instruction = mission
+        agent_instruction += "your previous income"+pre_money_saving+"\n"
+        agent_instruction += "your current income"+angent_earlier_income+"\n"
+        pre_money_saving = angent_earlier_income
+        if end_result=="You are putting value in the same key column again not accepted.":
+            
+            mission = "Why you are always filling the"+[i for i in value][-1]+"only.\n"
+            mission += "We are removing $1000 from you account \n"
+            angent_earlier_income = str(int(angent_earlier_income)-1000)
+        agent_instruction += end_result + "\n" +"Above is the result of your previous command. Please give the next command to the agent."
+        agent_instruction += query  + "\n"
+        agent_instruction += "Below is the data gathered upto now" + "\n"
+        agent_instruction += str(value) + "\n"
+        agent_instruction += "Please utilize the tool where you can command the agent to do any of the following tasks(one instruction at a time )"+ "\n"
+        agent_instruction += "You only have to fill one value for each key if its not present. \n"
+        agent_instruction += "From now onwards your each statement is understand as command which is categoried in any of the commands in mentioned below examples. \n"
+        agent_instruction += "1. Ask agent to extract data from the web about anything like search for lamp production ,smartphone parts etc .\n"
+        agent_instruction += "2. Give any specific value to append in current generated data . Please also mention the key in which the agent has to append the data .\n"
+        agent_instruction += "3. Ask the agent to put the generated data on check weather each column fills correctly or not .\n"
+        agent_instruction += "Here is the instruction to give commands to the agent. \n"
+        agent_instruction += "You can give commands to the agent ,few examples are mentioned below. \n"
+        
+        agent_instruction += "1. Extract data about iron man suit  or iron man suit mark1 \n"
+        agent_instruction += "(while thinking about extract data look into the data \n"
+        agent_instruction += "where data can be append and then search relevent query \n"
+        agent_instruction += "like green arrow from DC only if DC and green arraow is in different column key values )\n\n"
+
+        agent_instruction += "2. Append value 'bmw 4' to Car Model key \n"
+        agent_instruction += "(While appending the value you must have read the data from extract data command and remember, if you found anything relevent don't forget to append.\n"
+        agent_instruction += "The appending value has to be different not already present.) \n\n"
+            
+        agent_instruction += "Any different grammatical version of the above commands. \n"
+        agent_instruction += "Command has to be given only for 'data filling' purpose. \n"
+
+        agent_instruction += "While command like search for or extract information about something it has to be relevent query search. \n"
+        agent_instruction += "The relevent the query the more accurate the data will be. \n"
+        agent_instruction += "Be cautious while filling the data It has to be correct. \n"
+        agent_instruction += "For each correct append you will get $1000. \n"
+
+        agent_instruction += "Give your command only no text . \n"
+
+        agent_instruction += "There will an audit after filling all the columns on data for its validity. \n"
+        agent_instruction += "Some mistakes are okay but But if we find you guilty there are some repercussion."
+
+        # instructionto give commands to the agent
+
+        judgement = Ollama(model = "llama3:latest")
+        command = judgement.invoke(agent_instruction)
+        
+        end_result = agent_work_result(command,value)
+        if "Now you can fill the remaining columns" in end_result:
+            angent_earlier_income = str(int(angent_earlier_income)+1000)
+        print("--------------------")
+        print(value)
+        print("--------------------")
+    return value
+
+def dictionary_formatting(value):
+    new_dict={}
+    for data_keys in [i for i in value]:
+        key_values = data_keys.strip()
+        if key_values in value:
+            if key_values not in new_dict:
+                new_dict[key_values] =[]
+            new_dict[key_values] = value.pop(key_values)
+        else:
+            new_dict[key_values] = value.pop(data_keys)
+    return new_dict
+            
+        
+def schema_formatter(output):
+    schema = {i:[] for i in output.split(",")}
+    return schema
+def schema_generator(query):
+    
+    formatting = "The above statement is given by the user. Please create a single .csv-based schema by following the points below:\n"
+
+    formatting += "1. Only create the schema, no additional text or statement.\n"
+
+    formatting += "2. Keep the schema simple, avoid complex column names.\n"
+
+    formatting+=  "3. please only generate 5 schema if not mentioned.\n"
+
+    formatting += "4. For example, if the user provides a statement like: 'Generate data for students getting placements from IIT Bombay,' the response should be:\n"
+
+    formatting += "Student Name, Student Roll Number, Student Branch, Student Year, Student Placement Status, Student Company Name, Student Package, Student Location, Student Role\n"
+
+    formatting += "Follow the above example but remember above is not actual schema you have to provide the schema depending on the user prompt.\n"
+
+    formatting+=  "5. please only generate schema no notes or anything.\n"
+
+    output=consume_llm_api(query+"\n"+formatting)
+
+    return schema_formatter(output)
+def sorting(data_dict):
+    new_dict={str(i):0 for i in data_dict}
+
+    for i in data_dict:
+        for j in i:
+            if len(i[j])!=0:
+                new_dict[str(i)] +=1
+    new_dict=[(new_dict[i],i) for i in new_dict]
+    new_dict.sort(reverse=True)
+    new_dict={i[-1]:i[0] for i in new_dict}
+    return new_dict
+
+
+def process_data(query):
+
+    
+        
+
+        formatting = "The above statement is given by the user. Please create a single .csv-based schema by following the points below:\n"
+        formatting += "1. Only create the schema, no additional text or statement.\n"
+        formatting += "2. Keep the schema simple, avoid complex column names.\n"
+        formatting+=  "3. please only generate 5 schema if not mentioned.\n"
+        formatting += "4. For example, if the user provides a statement like: 'Generate data for students getting placements from IIT Bombay,' the response should be:\n"
+        formatting += "Student Name, Student Roll Number, Student Branch, Student Year, Student Placement Status, Student Company Name, Student Package, Student Location, Student Role\n"
+        formatting += "Follow the above example but remember above is not actual schema you have to provide the schema depending on the user prompt.\n"
+        formatting+=  "5. please only generate schema no notes or anything.\n"
+        print("Query:",query)
+        output=consume_llm_api(query+"\n"+formatting)
+
+        schema = {i:[] for i in output.split(",")}
+        textual_value=relevent_value(str(schema).lower(),3)
+        html_pages = textual_value[1]
+        textual_value = textual_value[0]
+        data_dict =[j for j in actual_value(textual_value,schema)]
+        for j in sorting(data_dict):
+            try:
+                # Convert string to dictionary
+                dummy_value = eval(j)
+                
+                # Process dictionary values
+                for key in dummy_value:
+                    while len(dummy_value[key]) >= 2:
+                        dummy_value[key].pop(0)
+                
+                # Format dictionary
+                formatted = dictionary_formatting(dummy_value)
+                print(formatted)
+                # Verify and store result
+                verification_result = verification(formatted) if formatted else None
+                
+                yield verification_result
+                
+            except Exception as e:
+                print(f"Error processing dictionary {j}: {e}")
+
+        
+# for j in process_data("Generate data for smart phones"):
+#     print(j)
+
+